【python爬虫】设计自己的爬虫 2. 数据保存封装 mongodb，mysql和elasticsearch

loyd32023-12-07 12:40

mongodb, mysql和elasticsearch 功能较相似，所以打算用一套接口将它们封装起来

基类StorageBase

如下：

csharp 复制代码 
class StorageBase:
    def __init__(self, host=None, port=None, database=None, table=None, location=None, account=None, password=None,
                 url=None):
        self.host = host
        self.port = port
        self.database = database
        self.table = table
        self.location = location
        self.account = account
        self.password = password
        self.url = url

    def build_connect(self):
        raise NotImplementedError

    # 增
    def add(self, table_collection_index, data, id=None):
        raise NotImplementedError

    # 删
    def delete(self, table_collection_index, condition=None, data=None, id=None):
        raise NotImplementedError

    # 改
    def update(self, table_collection_index, data=None, condition=None, id=None, update_condition=None):
        raise NotImplementedError

    # 查
    def search(self, table_collection_index, condition=None):
        raise NotImplementedError

    # 没有就新增，有就更新
    def add_or_update(self, table_collection_index, data, condition=None):
        raise NotImplementedError

封装mongodb

csharp 复制代码 
class MongodbStore(StorageBase):
    def __init__(self, host=MONGODB_CONNECTION_HOST, port=MONGODB_CONNECTION_PORT, database=MONGODB_DATABASE,
                 account=MONGODB_DATABASE_USER, password=MONGODB_DATABASE_PASS):
        self.client = None
        self.db = None

        StorageBase.__init__(self, host=host, port=port,
                             database=database, account=account, password=password)

    def build_connect(self):
        try:
            connection = f"mongodb://{self.account}:{self.password}@{self.host}:{self.port}/{self.database}"
            self.client = pymongo.MongoClient(connection)
            self.db = self.client[self.database]

        except Exception as e:
            print("失败：{0}".format(e))
            self.db = None
        else:
            print(f"连接成功")

        return self.db

    # 增
    # data要是列表形式，否则报错
    # 添加单个可以是 单个元素的数组
    def add(self, table_collection_index, data, id=None):
        try:
            result = self.db[table_collection_index].insert_many(data)
            print(f"新增成功")
        except Exception as e:
            print("失败：{0}".format(e))
            result = None
        else:
            print(f"新增成功")

        return result

    def add_one(self, table_collection_index, data):
        try:
            result = self.db[table_collection_index].insert_one(data)

        except Exception as e:
            print("失败：{0}".format(e))
            result = None
        else:
            print(f"新增成功")
        return result

    # 没有就新增，有就更新
    def add_or_update(self, table_collection_index, data, condition=None):
        existing_doc = self.search_one(table_collection_index, conditions=None)

        if existing_doc:
            # 如果文档存在，则更新

            self.update_one(table_collection_index, condition, {
                "key": "set",
                "value": data
            })
            print('数据已更新。')
        else:
            # 如果文档不存在，则插入新文档
            self.add_one(table_collection_index, data)
            print('新数据已插入。')

    # 删
    def delete(self, table_collection_index, condition=None, data=None, id=None):
        try:
            result = self.db[table_collection_index].delete_many(parse_mongodb_condition(data))

        except Exception as e:
            print("失败：{0}".format(e))
            result = None
        else:
            print(f"删除成功")
        return result

    def delete_one(self, table_collection_index, data):
        try:
            result = self.db[table_collection_index].delete_one(parse_mongodb_condition(data))

        except Exception as e:
            print("失败：{0}".format(e))
            result = None
        else:
            print(f"删除成功")
        return result

    # 改
    # search_condition 指定查询条件
    # update_condition 指定更新条件

    def update(self, table_collection_index, data=None, condition=None, id=None, update_condition=None):
        try:
            result = self.db[table_collection_index].update_many(parse_mongodb_condition(condition),
                                                                 parse_mongodb_condition(update_condition))
        except Exception as e:
            logging.error(f"失败：{e}")
            result = None
        else:
            print(
                f"匹配的数据{result.matched_count if result else 0}条，影响的数据{result.modified_count if result else 0}条")

        return result

    # 改单个
    def update_one(self, table_collection_index=None, search_condition=None, update_condition=None):

        try:
            result = self.db[table_collection_index].update_one(parse_mongodb_condition(search_condition),
                                                                parse_mongodb_condition(update_condition))

        except Exception as e:
            print("失败：{0}".format(e))
            result = None
        else:
            print(
                f"匹配的数据{result.matched_count if result else 0}条，影响的数据{result.modified_count if result else 0}条")

        return result

    # 计数
    def count(self, table_collection_index, conditions):

        try:
            result = self.db[table_collection_index].count_documents(parse_mongodb_condition(conditions))

        except Exception as e:
            print("失败：{0}".format(e))
            result = None
        else:
            print(f"共有{result.matched_count if result else 0}条数据")

        return result

    # 查
    def search(self, table_collection_index, conditions=None):

        try:
            results = self.db[table_collection_index].find(parse_mongodb_condition(conditions) if conditions else {})

        except Exception as e:
            print("失败：{0}".format(e))
            results = None

        return results

    def search_one(self, table_collection_index, conditions=None):

        try:
            result = self.db[table_collection_index].find_one(parse_mongodb_condition(conditions) if conditions else {})
            print(f"查询到的内容：{result}")
        except Exception as e:
            print("失败：{0}".format(e))
            result = None

        return result

    # 排序
    def sort(self, table_collection_index, sort_key, conditions=None, skip_num=None, limit_num=None,
             sort_type=pymongo.ASCENDING):

        try:
            query = self.db[table_collection_index].find(
                parse_mongodb_condition(conditions) if conditions else {}).sort(sort_key,
                                                                                sort_type)
            if skip_num and limit_num:
                results = query.skip(skip_num).limit(limit_num)
            elif skip_num:
                results = query.skip(skip_num)
            elif limit_num:
                results = query.limit(limit_num)
            else:
                results = query

        except Exception as e:
            print("失败：{0}".format(e))
            result = None

        return results

# 测试代码
if __name__ == "__main__":
    # 建立连接
    mongodb_connect = MongodbStore()
    mongodb_connect.build_connect()

    # student1 = {
    #     "id": 1,
    #     "name": "Jordan3",
    #     "age": 23,
    #     "gender": "male"
    # }
    #
    # student2 = {
    #     "name": "Mike",
    #     "age": 21,
    #     "gender": "male"
    # }
    # mongodb_connect.add("students", [student1])

    # result = mongodb_connect.delete("students", {"name": "Mike"})
    # result = mongodb_connect.update("students", {"age": {"#gt": 23}}, {"$inc": {"age": 1}})
    # print(result.matched_count, result.modified_count)

    # 自定义分隔符号
    # condition = parse_mongodb_condition("age*>*20;name*regex*'^J.*'", ";", "*")

    condition_list = [
        {
            "key": "age",
            "condition": ">",
            "value": "20"
        },
        {
            "key": "name",
            "condition": "regex",
            "value": "'^J.*'"
        },
    ]

    # 不设置condition的话表示等于
    # search_list = [
    #     {
    #         "key": "age",
    #         "value": "20"
    #     },
    # ]

    # 条件表达式解析
    mongodb_connect.search("students", condition_list)
    mongodb_connect.sort("students", 'age', condition_list, None, None, pymongo.DESCENDING)

然后用的字典表如下

csharp 复制代码 
MONGODB_CONDITION_DICTIONARY = {
    '<': '$lt',
    '>': '$gt',
    '<=': '$lte',
    '>=': '$gte',
    '!=': '$ne',
    'in': '$in',
    'notin': '$nin',
    'regex': '$regex',
    'exists': '$exists',
    'type': '$type',
    'mod': '$mod',
    'inc': '$inc',
    'search': '$search',
    'text': '$text',
    'set': '$set',
}

这里要注意的是考虑到mongodb搜索条件的语法有些复杂，因此对其进行了一些简化，简化的思路是把条件转化成简单的键值对对象，下面看代码

csharp 复制代码 
# 优化该方法的思路
# 1. 使用列表推导式替代了外部循环。
# 2. 使用 str.join() 方法替代了内部循环中的字符串连接操作，这样可以减少不必要的字符串创建和连接操作，提高代码效率。
# 3. 使用 dict.values() 方法直接获取字典中的所有值，避免了在内部循环中通过键获取值的操作。
# 将条件对象数组转化为条件字符串
def turn_list_to_condition_str(list, split_cond=',', split_key=':'):
    conditions_list = [
        split_key.join([str(value) for value in search.values()])
        for search in list
    ]
    return split_cond.join(conditions_list)



# 该方法将条件字符串转换为条件对象
def parse_mongodb_condition(conditions, split_cond=',', split_key=':'):
    # 传入字符串规则
    # 例子："age:>:20,name:regex:'^J.*'"
    # ,分隔多个条件
    # age:>:20  age是字段 >是条件 20是值 中间用:分隔
    # 分隔符默认是,和: 也可以自定义
    # 分割字符串

    pairs = turn_list_to_condition_str(conditions).split(split_cond)

    # 构建字典
    condition_dict = {}

    # 避免了在每次循环中都多次调用 pair.split(split_key)
    for pair in pairs:
        split_pair = pair.split(split_key)
        key = replace_chars_by_dit(split_pair[0], MONGODB_CONDITION_DICTIONARY)

        if len(split_pair) == 3:
            cond = replace_chars_by_dit(split_pair[1], MONGODB_CONDITION_DICTIONARY)
            condition_dict[key] = {cond: eval(split_pair[2])}

        elif len(split_pair) == 2:
            condition_dict[key] = eval(split_pair[1])

    # print(f'解析后的条件表达式是{condition_dict}')
    return condition_dict

# 测试代码
if __name__ == "__main__":
    # condition_str = "age:20,name:regex:'^J.*'"
    # print(parse_mongodb_condition(condition_str))

    search_list = [
        {
            "key": "age",
            "condition": ">",
            "value": "20"
        },
        {
            "key": "name",
            "condition": "regex",
            # 注意字符串的值要帶单引号''
            "value": "'^J.*'"
        },
    ]

    search_list = {
        "key": "set",
        "value": "20"
    },
    print(parse_mongodb_condition(search_list))

封装 Elasticsearch

csharp 复制代码 
# 可以和关系数据库的概念对比理解
# Relational DB -> Databases -> Tables -> Rows -> Columns
# Elasticsearch -> Indices -> Types -> Documents -> Fields
class ElasticsearchStore(StorageBase):
    def __init__(self, host=ELASTICSEARCH_HOST, port=ELASTICSEARCH_PORT,
                 account=ELASTICSEARCH_USERNAME, password=ELASTICSEARCH_PASSWORD,
                 verify_certs=ELASTICSEARCH_VERIFY_CERTS):

        self.es = None
        self.verify_certs = verify_certs

        StorageBase.__init__(self, host=host, port=port, account=account, password=password)

    def build_connect(self, default=True):
        try:
            if default:
                self.es = Elasticsearch()
            else:
                url = f"https://[{self.account}:{self.password}@]{self.host}:{self.port}"
                self.es = Elasticsearch(url, verify_certs=self.verify_certs)

        except Exception as e:
            print("失败：{0}".format(e))
            self.es = None
        else:
            print(f"连接成功")

        return self.es

    # 增
    def add(self, table_collection_index, data, id=None):
        try:
            result = self.es.index(index=table_collection_index, body=data, id=id)
            print(f"新增成功")
        except Exception as e:
            print("失败：{0}".format(e))
            result = None
        else:
            print(f"新增成功")

        return result

    # 删
    def delete(self, table_collection_index, condition=None, data=None, id=None):
        try:
            result = self.es.delete(index=table_collection_index, id=id, ignore=[400, 404])
            print(f"删除成功")
        except Exception as e:
            print("失败：{0}".format(e))
            result = None
        else:
            print(f"删除成功")

        return result

    # 改
    def update(self, table_collection_index, data=None, condition=None, id=None, update_condition=None):
        try:
            result = self.es.update(index=table_collection_index, body=data, id=id, ignore=[400, 404])
        except Exception as e:
            logging.error(f"失败：{e}")
            result = None
        else:
            print(f"更新成功")

        return result

    # 获取
    def get(self, table_collection_index, id):
        try:
            result = self.es.get(index=table_collection_index, id=id, ignore=[400, 404])
        except Exception as e:
            logging.error(f"失败：{e}")
            result = None
        else:
            print(f"获取成功{result}")

        return result

    # 查
    # dsl = {
    #     'query': {
    #         'match': {
    #             'title': '高考 圆梦'
    #         }
    #     }
    # }
    def search(self, table_collection_index, condition=None):
        try:
            results = self.es.search(index=table_collection_index, body=condition)
        except Exception as e:
            logging.error(f"失败：{e}")
            results = None
        else:
            print(f"搜索成功{results}")

        return results


# 测试代码
if __name__ == "__main__":
    # 建立连接
    elasticsearch_store = ElasticsearchStore()
    elasticsearch_store.build_connect()

    # doc = {
    #     'author': 'kimchy',
    #     'text': 'Elasticsearch: cool. bonsai cool.',
    #     'timestamp': datetime.now(),
    # }
    # elasticsearch_connect.add('test-index', doc, 1)
    # elasticsearch_connect.get('test-index',1)
    doc1 = {
        'author': 'kimchy',
        'text': 'test',
        'timestamp': datetime.now(),
    }
    elasticsearch_store.update('test-index', doc1, 1)
    # elasticsearch_store.delete('test-index', 1)
    elasticsearch_store.search('test-index', {"query": {"match_all": {}}})

封装Mysql

csharp 复制代码 
class MysqlStore(StorageBase):
    def __init__(self, host=MYSQL_HOST, port=MYSQL_PORT,
                 account=MYSQL_USER, password=MYSQL_PASSWORD, database=MYSQL_DATABASE):
        self.db = None
        self.cursor = None
        StorageBase.__init__(self, host=host, port=port, account=account, password=password, database=database)

    def build_connect(self) -> pymysql.cursors.Cursor:
        """
        建立数据库连接。

        Returns:
            pymysql.cursors.Cursor: 数据库游标对象，用于执行 SQL 语句。
        """
        try:
            self.db = pymysql.connect(host=self.host, user=self.account, password=self.password, port=self.port,
                                      database=self.database)
            self.cursor = self.db.cursor()

        except Exception as e:
            print(f"连接失败：{e}")
            self.db = None
            self.cursor = None
        else:
            print("连接成功")

        return self.cursor

    # 增
    # table 表
    # data 新增数据 键值对对象
    def add(self, table_collection_index, data, id=None):
        # data.keys()返回的是键的数组
        keys = ', '.join(data.keys())
        # 下面这段是构造多个%s最为占位符，有几个字段就构造几个
        values = ', '.join(['%s'] * len(data))
        # 要执行的sql语句
        sql = 'INSERT INTO {table}({keys}) VALUES ({values})'.format(table=table_collection_index, keys=keys, values=values)
        try:
            self.cursor.execute(sql, tuple(data.values()))
            self.db.commit()
            print(f"新增成功")
        except Exception as e:
            print("失败：{0}".format(e))
            self.db.rollback()
        finally:
            print(f"{sql}")

    # 没有新增，有就更新
    # table 表
    # data 新增数据 键值对对象
    def add_or_update(self, table_collection_index, data, condition=None):
        # data.keys()返回的是键的数组
        keys = ', '.join(data.keys())
        # 下面这段是构造多个%s最为占位符，有几个字段就构造几个
        values = ', '.join(['%s'] * len(data))
        # 要执行的sql语句
        # 如果主键已经存在，就执行更新操作
        sql = 'INSERT INTO {table}({keys}) VALUES ({values}) ON DUPLICATE KEY UPDATE '.format(table=table_collection_index, keys=keys,
                                                                                              values=values)
        update = ', '.join(["{key} = %s".format(key=key) for key in data])
        sql += update

        try:
            self.cursor.execute(sql, tuple(data.values()) * 2)
            self.db.commit()
            print(f"新增成功")
        except Exception as e:
            print("失败：{0}".format(e))
            self.db.rollback()
        finally:
            print(f"{sql}")

    # 删
    # table 表名
    # condition 删除条件
    def delete(self, table_collection_index, condition=None, data=None, id=None):
        sql = 'DELETE FROM  {table} WHERE {condition}'.format(table=table, condition=condition)
        try:
            self.cursor.execute(sql)
            self.db.commit()
            print(f"删除成功")
        except Exception as e:
            print("失败：{0}".format(e))
            self.db.rollback()
        finally:
            print(f"{sql}")

    # 改
    # table 表名
    # data 数据
    # condition 条件
    def update(self, table_collection_index, data=None, condition=None, id=None, update_condition=None):
        # data.keys()返回的是键的数组
        keys = ', '.join(data.keys())
        # 下面这段是构造多个%s最为占位符，有几个字段就构造几个
        values = ', '.join(['%s'] * len(data))
        # 要跟新的数据
        update = ','.join(["{key} = %s".format(key=key) for key in data])
        # 要执行的sql语句
        sql = 'UPDATE {table} SET {update} WHERE {condition}'.format(table=table_collection_index, update=update, condition=condition)

        try:
            self.cursor.execute(sql, tuple(data.values()))
            self.db.commit()
            print(f"更新成功")
        except Exception as e:
            print("失败：{0}".format(e))
            self.db.rollback()
        finally:
            print(f"{sql}")

    # 查
    def search(self, table_collection_index, condition=None):
        # 要执行的sql语句
        sql = 'SELECT * FROM {table} WHERE {condition}'.format(table=table_collection_index, condition=condition)

        try:
            self.cursor.execute(sql)
            # fetchall获取结果的所有数据
            results = self.cursor.fetchall()
            # fetchall得到的是二重元祖，其中每个元素都是一条记录
            # 要注意的是fetch的内部实现中有一个偏移指针，用来指向查询结果，偏移指针最开始指向第一条数据，取了一次数据后，指针偏移到下一条数据，
            # fetchone被调用后，结果的偏移指针会指向下一条数据，fetchall方法返回的是从偏移指针指向的数据一直到结束的所有数据，所有fetchall获取的数据会少一条

            for row in results:
                print(row)

        except Exception as e:
            print("失败：{0}".format(e))

# 测试代码
if __name__ == "__main__":
    # 建立连接
    mysql_connect = MysqlStore()
    mysql_connect.build_connect()
    data = {
        "name": "Bob",
        "age": 22
    }
    table = "students"
    mysql_connect.search(table, 'age>=20')

最后要注意的是

数据库连接的相关信息都放在统一的文件里并且设置为默认的值，也可以初始化的时候传入相应的值

相关推荐
傻啦嘿哟36 分钟前
如何使用 Python 开发一个简单的文本数据转换为 Excel 工具
开发语言·python·excel
B站计算机毕业设计超人43 分钟前
计算机毕业设计SparkStreaming+Kafka旅游推荐系统 旅游景点客流量预测 旅游可视化 旅游大数据 Hive数据仓库 机器学习 深度学习
大数据·数据仓库·hadoop·python·kafka·课程设计·数据可视化
IT古董1 小时前
【人工智能】Python在机器学习与人工智能中的应用
开发语言·人工智能·python·机器学习
不做超级小白1 小时前
工欲善其事，必先利其器；爬虫路上，我用抓包
爬虫
Smartdaili China1 小时前
如何在 Microsoft Edge 中设置代理： 快速而简单的方法
前端·爬虫·安全·microsoft·edge·社交·动态住宅代理
湫ccc2 小时前
《Python基础》之pip换国内镜像源
开发语言·python·pip
hakesashou2 小时前
Python中常用的函数介绍
java·网络·python
菜鸟的人工智能之路2 小时前
极坐标气泡图：医学数据分析的可视化新视角
python·数据分析·健康医疗
菜鸟学Python2 小时前
Python 数据分析核心库大全！
开发语言·python·数据挖掘·数据分析
小白不太白9502 小时前
设计模式之 责任链模式
python·设计模式·责任链模式
热门推荐
01(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明02PyTorch机器学习实现液态神经网络03【HarmonyOS】HUAWEI DevEco Studio 下载地址汇总04玄机平台应急响应—webshell查杀05Coze扣子平台完整体验和实践（附国内和国际版对比）06Ubuntu 20.04使用Livox mid 360 测试 FAST_LIO07【目标跟踪】相机运动补偿08Unity中PICO实现 隔空取物 和 接触抓取物体09RAG 实践- Ollama+RagFlow 部署本地知识库10怎样让音频速度变慢？请跟随以下方法进行操作