go并发请求url

复制代码
sync.WaitGroup写法
Go 复制代码
package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	//开始计时
	start := time.Now()

	//链接数据库,用户名:密码@tcp(地址:端口)/数据库名
	dsn := "gin:gin@tcp(42.1.25.18:3306)/gin"
	// 连接到MySQL数据库
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		fmt.Println("数据库连接错误: ", err)
		return
	}
	defer db.Close()

	urls := []string{
		"https://www.baidu.com",
		"https://www.taobao.com",
		"https://www.tmall.com",
		"https://www.jd.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://www.163.com",
		"https://www.zhihu.com",
		"https://www.weibo.com",
		"https://www.bilibili.com",
		"https://www.youku.com",
		"https://www.iqiyi.com",
		"https://www.alipay.com",
		"https://www.dingtalk.com",
		"https://www.wechat.com",
		"https://www.360.cn",
		"https://www.58.com",
		"https://www.liepin.com",
		"https://www.zhaopin.com",
		"https://www.douban.com",
		"https://www.ifeng.com",
		"https://www.ctrip.com",
		"https://www.qunar.com",
		"https://www.meituan.com",
		"https://www.dianping.com",
		"https://www.ele.me",
		"https://www.vip.com",
		"https://www.suning.com",
		"https://www.mi.com",
		"https://www.huawei.com",
		"https://www.vivo.com.cn",
		"https://www.oppo.com",
		"https://www.le.com",
		"https://www.ganji.com",
		"https://www.kuaidi100.com",
		"https://www.17173.com",
		"https://www.4399.com",
		"https://www.tgbus.com",
		"https://www.10010.com",
		"https://www.10086.cn",
		"https://www.189.cn",
		"https://www.21cn.com",
		"https://www.chinadaily.com.cn",
		"https://www.xinhuanet.com",
		"https://www.people.com.cn",
		"https://www.thepaper.cn",
		"https://www.guokr.com",
		"https://www.ftchinese.com",
		"https://www.cbnweek.com",
		"https://www.163.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://e.baidu.com",
		"https://hezuo.baidu.com",
		"https://yiyan.baidu.com",
	}

	//设置3s超时,避免请求时间过长
	client := http.Client{
		Timeout: 3 * time.Second,
	}

	//定义chan结构体,url,status
	type Web struct {
		url    string
		status string
	}

	//并发请求,定义一个WaitGroup,用于等待所有请求完成
	var wg sync.WaitGroup

	for _, url := range urls {
		//增加一个等待
		wg.Add(1)
		//启动一个goroutine
		go func(url string) {
			//协程结束时调用Done通知main函数工作已经完成
			defer wg.Done()
			//请求
			resp, err := client.Get(url)
			if err != nil {
				fmt.Println("请求失败", err)
				_, err := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", url, "请求失败")
				if err != nil {
					return
				}
			} else {
				// 写入数据库
				_, err1 := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", url, resp.StatusCode)
				if err1 != nil {
					fmt.Println("写入数据库错误:", err1)
					return
				}
			}
		}(url)
	}

	//等待所有请求完成
	wg.Wait()

	//输出耗时
	elapsed := time.Since(start)
	fmt.Println("耗时:", elapsed)
}

sync.WaitGroup+channel

Go 复制代码
package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	//开始计时
	start := time.Now()

	//链接数据库,用户名:密码@tcp(地址:端口)/数据库名
	dsn := "gin:gin@tcp(4.19.21.17:3306)/gin"
	// 连接到MySQL数据库
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		fmt.Println("数据库连接错误: ", err)
		return
	}
	//关闭数据库
	defer db.Close()

	urls := []string{
		"https://www.baidu.com",
		"https://www.taobao.com",
		"https://www.tmall.com",
		"https://www.jd.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://www.163.com",
		"https://www.zhihu.com",
		"https://www.weibo.com",
		"https://www.bilibili.com",
		"https://www.youku.com",
		"https://www.iqiyi.com",
		"https://www.alipay.com",
		"https://www.dingtalk.com",
		"https://www.wechat.com",
		"https://www.360.cn",
		"https://www.58.com",
		"https://www.liepin.com",
		"https://www.zhaopin.com",
		"https://www.douban.com",
		"https://www.ifeng.com",
		"https://www.ctrip.com",
		"https://www.qunar.com",
		"https://www.meituan.com",
		"https://www.dianping.com",
		"https://www.ele.me",
		"https://www.vip.com",
		"https://www.suning.com",
		"https://www.mi.com",
		"https://www.huawei.com",
		"https://www.vivo.com.cn",
		"https://www.oppo.com",
		"https://www.le.com",
		"https://www.ganji.com",
		"https://www.kuaidi100.com",
		"https://www.17173.com",
		"https://www.4399.com",
		"https://www.tgbus.com",
		"https://www.10010.com",
		"https://www.10086.cn",
		"https://www.189.cn",
		"https://www.21cn.com",
		"https://www.chinadaily.com.cn",
		"https://www.xinhuanet.com",
		"https://www.people.com.cn",
		"https://www.thepaper.cn",
		"https://www.guokr.com",
		"https://www.ftchinese.com",
		"https://www.cbnweek.com",
		"https://www.163.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://e.baidu.com",
		"https://hezuo.baidu.com",
		"https://yiyan.baidu.com",
	}

	//设置3s超时,避免请求时间过长
	client := http.Client{
		Timeout: 3 * time.Second,
	}

	//定义chan结构体,url,status
	type Web struct {
		url    string
		status string
	}

	//并发请求
	var wg sync.WaitGroup
	//创建一个并发请求的channel,缓冲区大小为urls的长度
	results := make(chan *Web, len(urls))

	//遍历urls,k为索引,url为值
	for k, url := range urls {
		//增加一个等待
		wg.Add(1)
		fmt.Println("开始请求:", k, url)
		//启动一个goroutine,传入k和url
		go func(k int, url string) {
			//协程结束时调用Done通知main函数工作已经完成
			defer wg.Done()
			//请求
			resp, err := client.Get(url)
			if err != nil {
				fmt.Println("请求失败", err)
				//设置状态为请求失败
				results <- &Web{url: url, status: "请求失败"}
				return
			}
			//关闭请求
			defer resp.Body.Close()
			fmt.Println("请求完成:", k, url)
			//写入channel
			results <- &Web{url: url, status: strconv.Itoa(resp.StatusCode)}
		}(k, url)
	}

	//统计有多少个请求完成了
	fmt.Println("等待所有请求完成")
	//等待所有请求完成
	wg.Wait()
	//关闭channel
	close(results)
	//统计results的长度
	//fmt.Println("results长度:", len(results))

	fmt.Println("开始写入数据库")
	//遍历channel
	for resp := range results {
		// 写入数据库
		_, err := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", resp.url, resp.status)
		if err != nil {
			fmt.Println("写入数据库错误:", err)
		}
	}
	fmt.Println("写入数据库完成")

	//输出耗时
	elapsed := time.Since(start)
	fmt.Println("耗时:", elapsed)
}
相关推荐
行百里er22 分钟前
【Spring Insight 技术分享】runtime-settings.json 的妙用
后端·开源·监控
whitelbwwww42 分钟前
CUDA C++实战--中级
开发语言·前端·c++
FungLeo1 小时前
FengCMS · PHP+MySQL 升级实录 · 内容写入一致性:锁等待超时与跨模型子树删除竞态
开发语言·mysql·php·工程实践·fengcms
databook1 小时前
处理一批 PDF,怎么算真的处理完了
后端·python
苏三说技术2 小时前
为什么越来越多人用 Docling ? - 苏三说技术
后端
nice_lcj5202 小时前
【从零理解计算机网络 · 12】登录态的载体:Cookie、Session、Token 在网络层怎么走
后端·计算机网络·面试·jwt·token·cookie·session
谙弆悕博士3 小时前
多智能体架构详解:市场拍卖式架构(Market-based / 竞价协商)
开发语言·架构·去中心化·多智能体·智能体·拍卖架构
ShineWinsu3 小时前
对于Qt:QMessageBox的解析
开发语言·c++·qt·面试·客户端·qmessagebox·消息对话框
承渊政道3 小时前
【Python编程快速上手—让繁琐工作自动化】(列表:把数据排成队,从下标取值到深拷贝)
开发语言·python·pycharm·自动化·列表
考虑考虑4 小时前
JDK27中的加密对象的 PEM 编码
java·后端·java ee