python
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
import csv
from tqdm import tqdm
# 设置Chrome选项,使其在无头模式下运行
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--no-sandbox')
# 禁止加载图片
prefs = {"profile.managed_default_content_settings.images": 2}
chrome_options.add_experimental_option("prefs", prefs)
# CSV文件名
csv_file = 'lists_3.csv'
# 指定ChromeDriver的路径
driver_path = '/usr/bin/chromedriver'
# 创建一个Service对象
service = Service(driver_path)
# 创建WebDriver实例
driver = webdriver.Chrome(service=service, options=chrome_options)
# 从CSV文件读取URL列表
urls = []
with open(csv_file, mode='r', encoding='utf-8') as file:
csv_reader = csv.reader(file)
for row in csv_reader:
if row: # 确保行不是空的
urls.append(row[0]) # 假设URL在每行的第一个元素
# 输出文件
output_file = 'code.txt'
# 使用'a'模式打开输出文件
with open(output_file, 'a', encoding='utf-8') as file:
for url in tqdm(urls, desc="Processing URLs"):
try:
# 访问URL
driver.get(url)
# 显式等待页面标题出现
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "title")))
# 获取页面标题
title = driver.title
# 尝试获取<title>,并写入文件
file.write(f"{url}\n")
file.write(f"Title: {title}\n")
# 尝试获取<meta name="keywords">
try:
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.NAME, "keywords")))
keywords = driver.find_element(By.NAME, 'keywords').get_attribute('content')
file.write(f"Keywords: {keywords}\n")
except Exception:
file.write("Keywords: Not Found\n")
# 尝试获取<meta name="description">
try:
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.NAME, "description")))
description = driver.find_element(By.NAME, 'description').get_attribute('content')
file.write(f"Description: {description}\n")
except Exception:
file.write("Description: Not Found\n")
file.write("\n") # 分隔不同URL的元数据
file.flush() # 强制刷新文件缓冲
except Exception as e:
print(f"An error occurred while processing {url}: {str(e)}")
continue # 跳过错误URL,继续处理下一个
# 关闭WebDriver实例
driver.quit()
print("All data has been written to the output file.")