go并发请求url - 技术栈

复制代码

sync.WaitGroup写法

Go 复制代码

package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	//开始计时
	start := time.Now()

	//链接数据库，用户名：密码@tcp(地址:端口)/数据库名
	dsn := "gin:gin@tcp(42.1.25.18:3306)/gin"
	// 连接到MySQL数据库
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		fmt.Println("数据库连接错误: ", err)
		return
	}
	defer db.Close()

	urls := []string{
		"https://www.baidu.com",
		"https://www.taobao.com",
		"https://www.tmall.com",
		"https://www.jd.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://www.163.com",
		"https://www.zhihu.com",
		"https://www.weibo.com",
		"https://www.bilibili.com",
		"https://www.youku.com",
		"https://www.iqiyi.com",
		"https://www.alipay.com",
		"https://www.dingtalk.com",
		"https://www.wechat.com",
		"https://www.360.cn",
		"https://www.58.com",
		"https://www.liepin.com",
		"https://www.zhaopin.com",
		"https://www.douban.com",
		"https://www.ifeng.com",
		"https://www.ctrip.com",
		"https://www.qunar.com",
		"https://www.meituan.com",
		"https://www.dianping.com",
		"https://www.ele.me",
		"https://www.vip.com",
		"https://www.suning.com",
		"https://www.mi.com",
		"https://www.huawei.com",
		"https://www.vivo.com.cn",
		"https://www.oppo.com",
		"https://www.le.com",
		"https://www.ganji.com",
		"https://www.kuaidi100.com",
		"https://www.17173.com",
		"https://www.4399.com",
		"https://www.tgbus.com",
		"https://www.10010.com",
		"https://www.10086.cn",
		"https://www.189.cn",
		"https://www.21cn.com",
		"https://www.chinadaily.com.cn",
		"https://www.xinhuanet.com",
		"https://www.people.com.cn",
		"https://www.thepaper.cn",
		"https://www.guokr.com",
		"https://www.ftchinese.com",
		"https://www.cbnweek.com",
		"https://www.163.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://e.baidu.com",
		"https://hezuo.baidu.com",
		"https://yiyan.baidu.com",
	}

	//设置3s超时，避免请求时间过长
	client := http.Client{
		Timeout: 3 * time.Second,
	}

	//定义chan结构体，url，status
	type Web struct {
		url    string
		status string
	}

	//并发请求，定义一个WaitGroup，用于等待所有请求完成
	var wg sync.WaitGroup

	for _, url := range urls {
		//增加一个等待
		wg.Add(1)
		//启动一个goroutine
		go func(url string) {
			//协程结束时调用Done通知main函数工作已经完成
			defer wg.Done()
			//请求
			resp, err := client.Get(url)
			if err != nil {
				fmt.Println("请求失败", err)
				_, err := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", url, "请求失败")
				if err != nil {
					return
				}
			} else {
				// 写入数据库
				_, err1 := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", url, resp.StatusCode)
				if err1 != nil {
					fmt.Println("写入数据库错误:", err1)
					return
				}
			}
		}(url)
	}

	//等待所有请求完成
	wg.Wait()

	//输出耗时
	elapsed := time.Since(start)
	fmt.Println("耗时：", elapsed)
}

sync.WaitGroup+channel

Go 复制代码

package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	//开始计时
	start := time.Now()

	//链接数据库，用户名：密码@tcp(地址:端口)/数据库名
	dsn := "gin:gin@tcp(4.19.21.17:3306)/gin"
	// 连接到MySQL数据库
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		fmt.Println("数据库连接错误: ", err)
		return
	}
	//关闭数据库
	defer db.Close()

	urls := []string{
		"https://www.baidu.com",
		"https://www.taobao.com",
		"https://www.tmall.com",
		"https://www.jd.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://www.163.com",
		"https://www.zhihu.com",
		"https://www.weibo.com",
		"https://www.bilibili.com",
		"https://www.youku.com",
		"https://www.iqiyi.com",
		"https://www.alipay.com",
		"https://www.dingtalk.com",
		"https://www.wechat.com",
		"https://www.360.cn",
		"https://www.58.com",
		"https://www.liepin.com",
		"https://www.zhaopin.com",
		"https://www.douban.com",
		"https://www.ifeng.com",
		"https://www.ctrip.com",
		"https://www.qunar.com",
		"https://www.meituan.com",
		"https://www.dianping.com",
		"https://www.ele.me",
		"https://www.vip.com",
		"https://www.suning.com",
		"https://www.mi.com",
		"https://www.huawei.com",
		"https://www.vivo.com.cn",
		"https://www.oppo.com",
		"https://www.le.com",
		"https://www.ganji.com",
		"https://www.kuaidi100.com",
		"https://www.17173.com",
		"https://www.4399.com",
		"https://www.tgbus.com",
		"https://www.10010.com",
		"https://www.10086.cn",
		"https://www.189.cn",
		"https://www.21cn.com",
		"https://www.chinadaily.com.cn",
		"https://www.xinhuanet.com",
		"https://www.people.com.cn",
		"https://www.thepaper.cn",
		"https://www.guokr.com",
		"https://www.ftchinese.com",
		"https://www.cbnweek.com",
		"https://www.163.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://e.baidu.com",
		"https://hezuo.baidu.com",
		"https://yiyan.baidu.com",
	}

	//设置3s超时，避免请求时间过长
	client := http.Client{
		Timeout: 3 * time.Second,
	}

	//定义chan结构体，url，status
	type Web struct {
		url    string
		status string
	}

	//并发请求
	var wg sync.WaitGroup
	//创建一个并发请求的channel,缓冲区大小为urls的长度
	results := make(chan *Web, len(urls))

	//遍历urls，k为索引，url为值
	for k, url := range urls {
		//增加一个等待
		wg.Add(1)
		fmt.Println("开始请求：", k, url)
		//启动一个goroutine，传入k和url
		go func(k int, url string) {
			//协程结束时调用Done通知main函数工作已经完成
			defer wg.Done()
			//请求
			resp, err := client.Get(url)
			if err != nil {
				fmt.Println("请求失败", err)
				//设置状态为请求失败
				results <- &Web{url: url, status: "请求失败"}
				return
			}
			//关闭请求
			defer resp.Body.Close()
			fmt.Println("请求完成：", k, url)
			//写入channel
			results <- &Web{url: url, status: strconv.Itoa(resp.StatusCode)}
		}(k, url)
	}

	//统计有多少个请求完成了
	fmt.Println("等待所有请求完成")
	//等待所有请求完成
	wg.Wait()
	//关闭channel
	close(results)
	//统计results的长度
	//fmt.Println("results长度：", len(results))

	fmt.Println("开始写入数据库")
	//遍历channel
	for resp := range results {
		// 写入数据库
		_, err := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", resp.url, resp.status)
		if err != nil {
			fmt.Println("写入数据库错误:", err)
		}
	}
	fmt.Println("写入数据库完成")

	//输出耗时
	elapsed := time.Since(start)
	fmt.Println("耗时：", elapsed)
}