golang爬虫下载的代码:
下载程序的借鉴内容:
这个是关于gbk,utf8等相互转换的包
github.com/axgle/mahonia"
一、标准下载代码
package downloader
import (
"log"
"net/http"
"io"
"github.com/axgle/mahonia"
"github.com/go-crawler/car-prices/fake"
)
func Get(url string) io.Reader {
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("http.NewRequest err: %v", err)
}
req.Header.Add("User-Agent", fake.GetUserAgent())
req.Header.Add("Referer", "https://car.autohome.com.cn")
resp, err := client.Do(req)
if err != nil {
log.Printf("client.Do err: %v", err)
}
mah := mahonia.NewDecoder("gbk")
return mah.NewReader(resp.Body)
}