github公开项目爬取

import requests


def search_github_repositories(keyword, token=None, language=None, max_results=1000):
    """
    通过 GitHub API 搜索仓库，支持分页获取所有结果（最多 1000 条）
    :param keyword: 搜索关键词
    :param token: GitHub Token（可选，但建议使用以提高速率限制）
    :param language: 过滤语言
    :param max_results: 最大结果数（GitHub 允许最多 1000 条）
    :return: 仓库列表
    """
    url = "https://api.github.com/search/repositories"
    headers = {"Accept": "application/vnd.github.v3+json"}
    if token:
        headers["Authorization"] = f"Bearer {token}"

    query = keyword
    if language:
        query += f" language:{language}"

    repositories = []
    page = 1
    per_page = 100  # GitHub 单页最大允许 100 条

    while len(repositories) < max_results:
        params = {
            "q": query,
            "sort": "stars",
            "order": "desc",
            "page": page,
            "per_page": per_page
        }

        try:
            response = requests.get(url, headers=headers, params=params)
            response.raise_for_status()
            data = response.json()

            # 检查是否还有更多结果
            if not data.get("items"):
                break

            for item in data["items"]:
                repo_info = {
                    "name": item["name"],
                    "owner": item["owner"]["login"],
                    "url": item["html_url"],
                    "clone_url": item["clone_url"],
                    "description": item["description"],
                    "language": item["language"],
                    "stars": item["stargazers_count"]
                }
                repositories.append(repo_info)
                # 达到用户指定的最大数量时停止
                if len(repositories) >= max_results:
                    break

            page += 1
            # GitHub 最多允许 10 页（即 10 * 100=1000 条）
            if page > 10:
                break

        except requests.exceptions.RequestException as e:
            print(f"请求失败: {e}")
            break

    return repositories


def save_to_txt(results, filename="github_results.txt"):
    """
       将结果保存到文本文件
       :param results: 仓库列表
       :param filename: 保存文件名
       """
    with open(filename, "w", encoding="utf-8") as f:
        for repo in results:
            # 格式化单行文本（用 | 分隔关键信息）
            line = (
                f"仓库: {repo['owner']}/{repo['name']} | "
                f"URL: {repo['url']} | "
                f"语言: {repo['language']} | "
                f"星数: {repo['stars']} | "
                f"克隆地址: {repo['clone_url']}\n"
            )
            f.write(line)
    print(f"结果已保存至 {filename}")


# 示例用法
if __name__ == "__main__":
    keyword = "Aerospace Control"
    language = "Python"
    token = "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"  # 强烈建议使用 Token

    # 获取最多 1000 条结果（实际数量取决于搜索匹配的总数）
    results = search_github_repositories(keyword, token, language=language, max_results=1000)
    if results:
        print(f"找到 {len(results)} 个 {language} 相关仓库:")
        for idx, repo in enumerate(results, 1):
            print(f"\n{idx}. {repo['owner']}/{repo['name']}")
            print(f"   URL: {repo['url']}")
            print(f"   语言: {repo['language']}")  # 显示语言
            print(f"   克隆地址: {repo['clone_url']}")
    else:
        print("未找到结果")
    if results:
        save_to_txt(results)
        print(f"实际获取 {len(results)} 条结果")
    else:
        print("未找到结果")