python爬虫抓取新闻并且植入自己的mysql远程数据库内

python爬虫抓取新闻并且植入自己的mysql远程数据库内!这个代码是我自己写了很久才写好的,分享给大家。喜欢的点个赞。


复制代码
# -*- coding: utf-8 -*-
from xml.etree import ElementTree as ET
import datetime
import random

import pymysql
from selenium import webdriver
from lxml import etree
from time import sleep
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By


def strreplace_v1(old_str, key, value):
    # 替换某个字符串的一个或某几个字符串
    new_str = old_str.replace(key, value)
    return new_str


def get_page_source_html(driver, urlinfo):
    driver.get(urlinfo)
    page_text = driver.page_source
    tree = etree.HTML(page_text)
    return tree


def get_page_source_etree(driver):
    page_text = driver.page_source
    tree = etree.HTML(page_text)
    return tree


def get_list_a(etree, xpathinfo):
    return etree.xpath(xpathinfo)


def get_news_title(etree, xpathino):
    return etree.xpath(xpathino)


def get_news_content(etree, xpathino):
    return etree.xpath(xpathino)


def get_news_publish(etree, xpathino):
    return etree.xpath(xpathino)


def getUA():
    uaList = [
        # 360
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
        # chrome
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36",
        # "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",

        # firefox
        # "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
        "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:36.0) Gecko/20100101 Firefox/36.0",

        # ie11
        # "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
        # ie8
        # "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; 4399Box.1357; 4399Box.1253; 4399Box.1357)",

        # 2345王牌
        # "Chrome/39.0.2171.99 Safari/537.36 2345Explorer/6.5.0.11018",

        # 搜狗
        # "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
        # opera
        "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60"

    ]
    headers = random.choice(uaList)
    return headers


def get_desurl_list():
    # 所有的目标url集合。500个。
    urlinfo_list = []
    tree = ET.parse('sitemap.xml')
    url = tree.find("url")
    for rank in tree.iter('loc'):
        urlinfo_list.append(rank.text)
    return urlinfo_list


def createwailian(urlwllist, urlzhiru):
    str = strreplace_v1(urlzhiru, 'hrefinfo', urlwllist[random.randint(0, len(urlwllist))])
    return str


if __name__ == "__main__":
    allwlurllist = get_desurl_list()
    options = Options()
    options.add_argument("--disable-desktop-notifications")
    options.add_argument('User-Agent=%s' % getUA())
    # options.add_argument('--proxy-server={0}'.format('103.37.141.69:80'))
    # 创建浏览器对象
    driver = webdriver.Chrome(options=options)
    urlend = ""
    urlbegin = 'http://www.106ms.com/index.php?list=6-'
    for urlstart in range(1, 10):
        print('当前正访问:{0}'.format(urlbegin + str(urlstart)))
        driver.get(urlbegin + str(urlstart))
        # //*[@id="menu-item-10"]/a
        # 点击SEO基础知识链接
        # driver.find_element(By.XPATH, value='//*[@id="menu-item-5"]/a').click()
        # 存储a标签的集合
        list_a = []
        # xpath获取到的集合是一个存储了大量的webelment对象的集合,想具体拿到属性信息,还得再写标签自身的xpath语法。
        # //*[@id="moar"]/section[2]/div/div[1]/div[1]/dl/dd/h3/a
        a_list = get_list_a(get_page_source_etree(driver), '//*[@id="moar"]/section[2]/div/div/div/dl/dd/h3/a')
        sleep(1)
        for a in a_list:
            href = a.xpath('./@href')[0]
            list_a.append(href)
        print("当前页面获取a标签集合长度为{0}".format(len(list_a)))
        sleep(1)
        # 遍历当前list_a
        try:
            db = pymysql.Connect(
                host='8.142.*.*',  # 服务器ip地址
                port=3306,  # mysql默认端口号
                user="106iiaa",  # 用户名
                password="yrdsrootadmi3",  # 密码
                charset="utf8",  # 字符集
                db="hbdsa89aa"  # 数据库
            )
            cursor = db.cursor()
            #xuanyan = "<p>本平台所发布的部分公开信息来源于互联网,转载的目的在于传递更多信息及用于网络分享,并不代表本站赞同其观点,本平台所提供的信息,只供参考之用。不保证信息的准确性、有效性、及时性和完整性。如有侵权请联系:[14878741214]删除,谢谢合作!</p>"
            #urlzhiru = '<p>网站入口1:<a href="hrefinfo" target="_blank">网站入口地址</a></p><p>网站入口2:<a href="http://diyigefan.com/" target="_blank">diyigefan.com</a></p>'
            readnumber = 0
            base_url = 'http://www.106ms.com'
            for newsurl in list_a:
                try:
                    wanquan = base_url + newsurl
                    # print(wanquan)
                    driver.get(wanquan)
                    # title = driver.find_element(By.CLASS_NAME, value='article-title').text
                    # content = driver.find_element(By.CLASS_NAME, value='article-content').text
                    # newscontent = driver.find_element(By.XPATH, value='//div[@class="news_txt"]')
                    #newscontentfordes = driver.find_element(By.CLASS_NAME, value='news_txt').text
                    # /html/head/meta[6]
                    newscontentfordes = driver.find_element(By.XPATH, value='/html/head/meta[6]')
                    # 使用beautifulsoup封装html源码信息,然后开始提取内容。
                    soup = BeautifulSoup(driver.page_source, features='lxml', from_encoding='utf-8')
                    for s in soup('img'):
                        s.extract()
                    for s in soup('a'):
                        s.extract()
                    # 已经提前清除了img标签和a标签了
                    allp = soup.find("article", {"class": "content text-left"}).findAll('div')
                    paragraphs = []
                    for x in allp:
                        paragraphs.append(str(x))
                    # 去掉最后一个元素的值。
                    content2 = ''.join(paragraphs[0:-1])
                    # print(content2)
                    # 标题信息
                    title = driver.find_element(By.XPATH, value='//*[@id="moar"]/section[2]/div/div/div/article/header/h2').text
                    keywords = title
                    # content = content2  + createwailian(allwlurllist, urlzhiru)
                    content = content2
                    # content = newscontent + xuanyan
                    des = str(newscontentfordes.strip())[0:120]
                    sql = 'insert into news (title, keywords, des,content, author,publish,click,state,attr,attrdiy,flag,cate,uid) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                    values = (
                        title, keywords, des, content, 'admin',
                        str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')),
                        random.randint(100, 999),
                        0, 0, 0, '画室新闻', 5, 3)
                    cursor.execute(sql, values)
                    db.commit()
                    readnumber = readnumber + 1
                    print("标题:{0}---插入数据库成功".format(title))
                except Exception as ee:
                    print('发生了异常', ee)
                    continue
        except Exception as e:
            # print('发生了异常', e)
            db.rollback()
        finally:
            cursor.close()
            db.close()
        sleep(5)

        print('本次任务成功植入{0}篇软文.'.format(readnumber))
        sleep(600)
    driver.quit()

前提是,你需要提前安装好里面使用到了一些插件包。

否则是会报错的。

相关推荐
灵智工坊LingzhiAI2 小时前
人体坐姿检测系统项目教程(YOLO11+PyTorch+可视化)
人工智能·pytorch·python
Hello.Reader5 小时前
Redis 延迟监控深度指南
数据库·redis·缓存
ybq195133454315 小时前
Redis-主从复制-分布式系统
java·数据库·redis
好奇的菜鸟8 小时前
如何在IntelliJ IDEA中设置数据库连接全局共享
java·数据库·intellij-idea
tan180°8 小时前
MySQL表的操作(3)
linux·数据库·c++·vscode·后端·mysql
满昕欢喜8 小时前
SQL Server从入门到项目实践(超值版)读书笔记 20
数据库·sql·sqlserver
Hello.Reader9 小时前
Redis 延迟排查与优化全攻略
数据库·redis·缓存
烛阴9 小时前
简单入门Python装饰器
前端·python
好开心啊没烦恼10 小时前
Python 数据分析:numpy,说人话,说说数组维度。听故事学知识点怎么这么容易?
开发语言·人工智能·python·数据挖掘·数据分析·numpy
面朝大海,春不暖,花不开10 小时前
使用 Python 实现 ETL 流程:从文本文件提取到数据处理的全面指南
python·etl·原型模式