最近在学习北京理工大学的爬虫课程，其中一个实例是讲如何爬取淘宝商品信息，现整理如下：
功能描述：获取淘宝搜索页面的信息，提取其中的商品名称和价格
探讨：淘宝的搜索接口
翻页的处理
技术路线:requests‐re
步骤：
1 分析搜索接口和翻页的URL对应属性（由于淘宝有反爬虫机制，我们事先需要先登录，绕过登录界面进行爬取）
在搜索框输入" 口罩 "：
每页有44个物品
然后打开开发者模式：
找到 cookie 和 user-agent
2 对于每个页面，提取商品名称和价格信息
查看其中一个URL链接的HTML文本：
可以看到商品价格和名称分别在：view_price ，raw_title字段，可以通过正则表达式进行匹配
3 将信息输出到屏幕上
源码如下：
import requestsimport re
def getHTMLText(url, header): try: r = requests.get(url, timeout=30, headers=header) r.raise_for_status() r.encoding = r.apparent_encoding return r.text except: return ""
def parsePage(ilt, html): try: plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html) tlt = re.findall(r'\"raw_title\"\:\".*?\"', html) for i in range(len(plt)): price = eval(plt[i].split(':')[1]) title = eval(tlt[i].split(':')[1]) ilt.append([price, title]) except: print("")
def printGoodsList(ilt): tplt = "{:4}\t{:8}\t{:16}" print(tplt.format("序号", "价格", "商品名称")) count = 0 for g in ilt: count = count + 1 print(tplt.format(count, g[0], g[1]))
def main(): goods = '口罩' depth = 3 start_url = 'https://s.taobao.com/search?q=' + goods infoList = [] for i in range(depth): try: url = start_url + '&s=' + str(44 * i) headers = { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36", "cookie": "t=3740175161b86e1af2b115b6841c32c0; lLtC1_=1; _samesite_flag_=true; cookie2=18420f1cd8e30d8e6109a1404a84002e; _tb_token_=353be1bb54db; thw=cn; _m_h5_tk=d606fe16c0e6a08db0b1ffb9d29470a4_1585735430453; _m_h5_tk_enc=d959a53bb548df2635d20f6336201e08; tk_trace=oTRxOWSBNwn9dPyorMJE%2FoPdY8zfvmw%2Fq5hp2NfBzP4gKCub6aQRjGnwgQfZeebgzqZLnVvfVfuxmOAckzdn8z9FwVFU5vP8FkLnLL4M%2B6wV%2B19uVAjEEctVyz3yypK%2FKCYnv6SeFz%2F4iyg7BAc1gTey%2BgmE%2BMVvCVuGPbG2nnVDBcE1wQ7rOt2RDOHkxf94LVgKemAaywxfQw3bGxWNCDcPH8fY8lcCzaRf7oVxEBlEuam5U4ubGkwuka26k%2FhWMPatkpRRx94y6pCZyoYEYCmS6uo%3D; enc=etROYELhyf0h8EMleJ%2BszwuKv35ybIdX5c8%2FHagXQjX0IW7RrIHyA0md%2BNjS%2BjkVA%2BWSBX2ZzOrRnQVrZOUbqg%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cna=XGtnFni+VC8CAdz5eaVgjDhI; unb=1591070119; lgc=%5Cu7231%5Cu7B11%5Cu6211; cookie17=UoTV7Xp9CdRoqw%3D%3D; dnk=%5Cu7231%5Cu7B11%5Cu6211; tracknick=%5Cu7231%5Cu7B11%5Cu6211; _l_g_=Ug%3D%3D; sg=%E6%88%919f; _nk_=%5Cu7231%5Cu7B11%5Cu6211; cookie1=Vv9PmpXZo6HhF57DVhpuis8jAofsZzXYV9agGn5u29w%3D; alitrackid=login.taobao.com; lastalitrackid=login.taobao.com; mt=ci=0_1; v=0; JSESSIONID=F0CE9C142B4C2D1675F031E2BC94D08A; tfstk=cM-CBsfsaQKalUNRfBMN8_CLRuSCacZCQv1MdFeA0XeKL09C6s4Q0sNCRLVTXZB1.; l=dBxLPJbIQ1IB9WDsBOCZVBLZhnQTGIRjDuSpQR8pi_5BW_QDDkbOo_p4QEp6DjWcMITB45hH9bwtMekbJPDsw9x6XaZTWaDDBOf..; isg=BExMEitIfhv0zmpsaPAdQs6PHax-hfAvx26vAaYNMfesMe07zpQ_v7XL0DkJKiiH; sgcookie=EWtS0eTKtWNu2wucfnvK6; uc1=existShop=false&cookie16=VFC%2FuZ9az08KUQ56dCrZDlbNdA%3D%3D&pas=0&cookie21=W5iHLLyFeYZ1WM9hVnmS&cookie14=UoTUP2WB4TWk0g%3D%3D&cookie15=WqG3DMC9VAQiUQ%3D%3D; uc3=id2=UoTV7Xp9CdRoqw%3D%3D&nk2=0%2B52brjO&vt3=F8dBxdAR%2Be8PPlDHvf8%3D&lg2=VFC%2FuZ9ayeYq2g%3D%3D; csg=8854d1ae; skt=115ba0bf3e955a86; existShop=MTU4NTczMTI4Nw%3D%3D; uc4=id4=0%40UOx%2FVb7AgnGmuIpUq4kquCOwwZ%2Fx&nk4=0%400VVfR45YSz5VcR2eSMu07Ro%3D; _cc_=U%2BGCWk%2F7og%3D%3D" } html = getHTMLText(url, headers) # print(html.text) parsePage(infoList, html) except: continue printGoodsList(infoList)
main()