目录
使用
工具有三个面板：每日消息、全局查询、更新数据库
"每日消息"和"全局查询"，数据源：同花顺7x24小时快讯
"更新数据库"，更新的数据源除"同花顺7x24小时快讯"，还有一些其他的数据源，这些数据没有并入每日消息和全局查询里，在本工具中只做了更新功能。本文会分享完整代码，大家要把多数据源合并到一起查询的，可自行更改。
每日消息
1 选择某个日期，下面列表中将显示该日期的快讯
2 数据库所包含快讯的时间区间
3 可以选择条件，筛选内容
4 左侧列表单机某行，会在右侧显现该信息内容
全局查询
1 全局查询，查询范围不局限在某日，而是根据条件在整个数据库中查询。未避免返回数据量太大，程序要求必须有查询条件。
2 左侧列表单机某行，会在右侧显现该信息内容
更新数据库
点击更新按钮，会更新相应的数据表
代码
数据库表创建
本文使用的数据库为PostgreSQL
t_ths7x24msg 存储 同花顺7x24小时快讯
create table t_ths7x24msg(
time_obj timestamp,
id varchar,
seq varchar,
title varchar,
digest text,
url varchar,
tag varchar,
ctime varchar,
rtime varchar,
source varchar,
picUrl varchar,
short text,
stock varchar[],
field varchar[],
tagInfo varchar[],
nature varchar,
import varchar,
time_str varchar
)
t_thsfinancemsg 存储同花顺财经资讯
create table t_thsfinancemsg(
belong_type varchar,
hash_code varchar,
time_obj timestamp,
timestamp_str varchar,
title varchar,
url varchar,
time_str varchar
)
代码-数据库相关操作
import psycopg2
def connect_db():
try:
conn = psycopg2.connect(database='db_stock',user='postgres',password='',host='127.0.0.1',port=5432)
except Exception as e:
print(f'connection failed。{e}')
else:
return conn
pass连接数据库
def query_duration_timestamp():
sql_str = '''
select min(time_obj),max(time_obj) from t_ths7x24msg;
'''
conn = connect_db()
cur = conn.cursor()
cur.execute(sql_str)
res = cur.fetchone()
cur.close()
conn.close()
return res
def query_max_timestamp():
sql_str = '''
select max(time_obj) from t_ths7x24msg;
'''
conn = connect_db()
cur = conn.cursor()
cur.execute(sql_str)
res = cur.fetchone()
cur.close()
conn.close()
return res
pass
def query_date_by_duration(left,right):
sql_str = f'''
select id,time_str,title,short,digest,url,stock,field,tagInfo from t_ths7x24msg where time_obj>=\'{left}\' and time_obj<=\'{right}\';
'''
conn = connect_db()
cur = conn.cursor()
cur.execute(sql_str)
res = cur.fetchall()
cur.close()
conn.close()
return res
pass
def query_filters_in_all():
sql_str = '''
select stock,field,tagInfo from t_ths7x24msg;
'''
conn = connect_db()
cur = conn.cursor()
cur.execute(sql_str)
res = cur.fetchall()
cur.close()
conn.close()
return res
pass
def query_data_by_query(where_str):
sql_str = f'''
select id,time_str,title,short,digest,url,stock,field,tagInfo from t_ths7x24msg where {where_str};
'''
conn = connect_db()
cur = conn.cursor()
cur.execute(sql_str)
res = cur.fetchall()
cur.close()
conn.close()
return res
pass
def query_max_timestamp_in_one_belongtype(belong_type:str):
sql_str = f"""
select max(time_obj) from t_thsfinancemsg where belong_type=\'{belong_type}\';
"""
conn = connect_db()
cur = conn.cursor()
cur.execute(sql_str)
res = cur.fetchone()
cur.close()
conn.close()
return res
def query_hashcode_in_maxtimestamp(time_str:str,belong_type:str):
sql_str = f"""
select hash_code from t_thsfinancemsg where belong_type=\'{belong_type}\' and time_obj>=\'{time_str}\';
"""
conn = connect_db()
cur = conn.cursor()
cur.execute(sql_str)
res = cur.fetchall()
cur.close()
conn.close()
return res代码-界面与操作逻辑
class NewsWidget(QWidget):
ONEDAY_TAB = '每日消息'
QUERY_TAB = '全局查询'
UPDATE_TAB = '更新数据库'
def __init__(self):
super().__init__()
self.setWindowTitle('消息工具')
self.setMinimumSize(QSize(1200,800))
self.tab_index_dict = {}
self.tabs = QTabWidget()
self.tabs.tabBarClicked.connect(self.tabs_tabBarClicked)
# tab1 start
widget_tab1 = QWidget()
self.label_title = QLabel('XXXX-XX-XX', alignment=Qt.AlignmentFlag.AlignHCenter)
self.label_title.setStyleSheet("font-size:28px;color:#CC2EFA;")
label00 = QLabel('切换日期')
self.dateedit = QDateEdit()
self.dateedit.setDisplayFormat('yyyy-MM-dd')
self.dateedit.userDateChanged.connect(self.dateedit_userDateChanged)
self.label_duration = QLabel('从 开始 到 结束')
self.label_duration.setMinimumWidth(300)
label01 = QLabel('股票',alignment=Qt.AlignmentFlag.AlignRight)
self.combo_stock = QComboBox()
label02 = QLabel('概念',alignment=Qt.AlignmentFlag.AlignRight)
self.combo_field = QComboBox()
label03 = QLabel('标记',alignment=Qt.AlignmentFlag.AlignRight)
self.combo_tag = QComboBox()
btn1_query = QPushButton('查询',clicked=self.btn1_query_clicked)
btn1_reset = QPushButton('重置',clicked=self.btn1_reset_clicked)
layout03 = QHBoxLayout()
layout03.addStretch(1)
layout03.addWidget(btn1_query)
layout03.addWidget(btn1_reset)
layout03.addStretch(1)
layout00 = QHBoxLayout()
layout00.addWidget(label01)
layout00.addWidget(self.combo_stock)
layout00.addWidget(label02)
layout00.addWidget(self.combo_field)
layout00.addWidget(label03)
layout00.addWidget(self.combo_tag)
groupbox = QGroupBox('筛选')
groupbox.setLayout(layout00)
layout01 = QHBoxLayout()
layout01.addStretch(1)
layout01.addWidget(label00)
layout01.addWidget(self.dateedit)
layout01.addWidget(self.label_duration)
layout01.addStretch(1)
layout02 = QVBoxLayout()
layout02.addWidget(self.label_title)
layout02.addLayout(layout01)
layout02.addWidget(groupbox)
layout02.addLayout(layout03)
widget_tab1.setLayout(layout02)
tab_index = self.tabs.addTab(widget_tab1,self.ONEDAY_TAB)
self.tab_index_dict[tab_index] = self.ONEDAY_TAB
# tab1 end
# tab2 start
widget_tab2 = QWidget()
label11 = QLabel('股票',alignment=Qt.AlignmentFlag.AlignRight)
self.combo2_stock = QComboBox()
label12 = QLabel('概念',alignment=Qt.AlignmentFlag.AlignRight)
self.combo2_field = QComboBox()
label13 = QLabel('标记',alignment=Qt.AlignmentFlag.AlignRight)
self.combo2_tag = QComboBox()
label14 = QLabel('模糊查询')
self.lineedit2_query = QLineEdit()
self.btn2_query = QPushButton('查询',clicked=self.btn2_query_clicked)
layout11 = QHBoxLayout()
layout11.addWidget(label11)
layout11.addWidget(self.combo2_stock)
layout11.addWidget(label12)
layout11.addWidget(self.combo2_field)
layout11.addWidget(label13)
layout11.addWidget(self.combo2_tag)
layout12 = QHBoxLayout()
layout12.addWidget(label14)
layout12.addWidget(self.lineedit2_query)
layout13 = QHBoxLayout()
layout13.addStretch(1)
layout13.addWidget(self.btn2_query)
layout13.addStretch(1)
layout14 = QVBoxLayout()
layout14.addLayout(layout11)
layout14.addLayout(layout12)
layout14.addLayout(layout13)
widget_tab2.setLayout(layout14)
tab_index = self.tabs.addTab(widget_tab2,self.QUERY_TAB)
self.tab_index_dict[tab_index] = self.QUERY_TAB
# tab2 end
# tab3 start
widget_tab3 = QWidget()
label30 = QLabel('同花顺7x24小时:')
self.btn3_update = QPushButton('更新-同花顺7x24小时',clicked=lambda :self.btn3_update_clicked('ths7x24',self.btn3_update))
layout22 = QGridLayout()
layout22.addWidget(label30, 0, 0, 1,1)
layout22.addWidget(self.btn3_update, 0, 1, 1,1)
label31 = QLabel('同花顺-滚动期货:')
self.btn3_update_gdqh = QPushButton('更新-滚动期货',clicked=lambda :self.btn3_update_clicked('ths_gdqh',self.btn3_update_gdqh))
layout23 = QGridLayout()
layout23.addWidget(label31, 0, 0, 1,1)
layout23.addWidget(self.btn3_update_gdqh, 0, 1, 1,1)
label32 = QLabel('同花顺-期市要闻')
self.btn3_update_qsyw = QPushButton('更新-期市要闻',clicked=lambda :self.btn3_update_clicked('ths_qsyw',self.btn3_update_qsyw))
layout22.addWidget(label32, 1, 0, 1,1)
layout22.addWidget(self.btn3_update_qsyw, 1, 1, 1,1)
label33 = QLabel('同花顺-商品期货')
self.btn3_update_spqh = QPushButton('更新-商品期货', clicked=lambda: self.btn3_update_clicked('ths_spqh',self.btn3_update_spqh))
layout23.addWidget(label33, 1, 0, 1,1)
layout23.addWidget(self.btn3_update_spqh, 1, 1, 1,1)
label34 = QLabel('同花顺-现货市场')
self.btn3_update_xhsc = QPushButton('更新-现货市场', clicked=lambda: self.btn3_update_clicked('ths_xhsc',self.btn3_update_xhsc))
layout22.addWidget(label34, 2, 0, 1,1)
layout22.addWidget(self.btn3_update_xhsc, 2, 1,1, 1)
label35 = QLabel('同花顺-财经要闻')
self.btn3_update_cjyw = QPushButton('更新-财经要闻', clicked=lambda: self.btn3_update_clicked('ths_cjyw',self.btn3_update_cjyw))
layout23.addWidget(label35, 2, 0, 1,1)
layout23.addWidget(self.btn3_update_cjyw, 2, 1, 1,1)
label36 = QLabel('同花顺-宏观经济')
self.btn3_update_hgjj = QPushButton('更新-宏观经济', clicked=lambda: self.btn3_update_clicked('ths_hgjj',self.btn3_update_hgjj))
layout22.addWidget(label36, 3, 0, 1,1)
layout22.addWidget(self.btn3_update_hgjj, 3, 1, 1,1)
label37 = QLabel('同花顺-产经新闻')
self.btn3_update_cjxw = QPushButton('更新-产经新闻', clicked=lambda: self.btn3_update_clicked('ths_cjxw',self.btn3_update_cjxw))
layout23.addWidget(label37, 3, 0, 1,1)
layout23.addWidget(self.btn3_update_cjxw, 3, 1, 1,1)
label38 = QLabel('同花顺-国际财经')
self.btn3_update_gjcj = QPushButton('更新-国际财经', clicked=lambda: self.btn3_update_clicked('ths_gjcj',self.btn3_update_gjcj))
layout22.addWidget(label38, 4, 0, 1,1)
layout22.addWidget(self.btn3_update_gjcj, 4, 1,1, 1)
label39 = QLabel('同花顺-金融市场')
self.btn3_update_jrsc = QPushButton('更新-金融市场', clicked=lambda: self.btn3_update_clicked('ths_jrsc',self.btn3_update_jrsc))
layout23.addWidget(label39, 4, 0,1, 1)
layout23.addWidget(self.btn3_update_jrsc, 4, 1,1, 1)
label40 = QLabel('同花顺-公司新闻')
self.btn3_update_gsxw = QPushButton('更新-公司新闻', clicked=lambda: self.btn3_update_clicked('ths_gsxw',self.btn3_update_gsxw))
layout22.addWidget(label40, 5, 0,1, 1)
layout22.addWidget(self.btn3_update_gsxw, 5, 1,1, 1)
label41 = QLabel('同花顺-区域经济')
self.btn3_update_qyjj = QPushButton('更新-区域经济', clicked=lambda: self.btn3_update_clicked('ths_qyjj',self.btn3_update_qyjj))
layout23.addWidget(label41, 5, 0,1, 1)
layout23.addWidget(self.btn3_update_qyjj, 5, 1,1, 1)
label42 = QLabel('同花顺-财经评论')
self.btn3_update_cjpl = QPushButton('更新-财经评论', clicked=lambda: self.btn3_update_clicked('ths_cjpl',self.btn3_update_cjpl))
layout22.addWidget(label42, 6, 0,1, 1)
layout22.addWidget(self.btn3_update_cjpl, 6, 1,1, 1)
label43 = QLabel('同花顺-财经人物')
self.btn3_update_cjrw = QPushButton('更新-财经人物', clicked=lambda: self.btn3_update_clicked('ths_cjrw',self.btn3_update_cjrw))
layout23.addWidget(label43, 6, 0,1, 1)
layout23.addWidget(self.btn3_update_cjrw, 6, 1,1, 1)
self.textedit3_log = QTextEdit()
layout21 = QHBoxLayout()
layout21.addLayout(layout22)
layout21.addLayout(layout23)
layout21.addWidget(self.textedit3_log)
widget_tab3.setLayout(layout21)
tab_index = self.tabs.addTab(widget_tab3,self.UPDATE_TAB)
self.tab_index_dict[tab_index] = self.UPDATE_TAB
# tab3 end
self.label_num = QLabel('几条')
self.table = QTableWidget()
self.table.setColumnCount(7)
self.table.setHorizontalHeaderLabels(['id','时间','title','short','stock','field','tag'])
self.table.setWordWrap(True)
self.table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers)
self.table.setSelectionMode(QAbstractItemView.SelectionMode.SingleSelection)
self.table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows)
self.table.itemClicked.connect(self.table_itemClicked)
label000 = QLabel('标题',alignment=Qt.AlignmentFlag.AlignRight)
self.lineedit000_title = QLineEdit()
label001 = QLabel('内容',alignment=Qt.AlignmentFlag.AlignRight)
self.textedit000_content = QTextEdit()
label002 = QLabel('原文链接',alignment=Qt.AlignmentFlag.AlignRight)
self.lineedit002_url = QLineEdit()
layout000 = QVBoxLayout()
layout000.addWidget(self.label_num)
layout000.addWidget(self.table)
layout001 = QFormLayout()
layout001.addRow(label000,self.lineedit000_title)
layout001.addRow(label001,self.textedit000_content)
layout001.addRow(label002,self.lineedit002_url)
layout002 = QHBoxLayout()
layout002.addLayout(layout000,3)
layout002.addLayout(layout001,1)
layout = QVBoxLayout()
layout.addWidget(self.tabs,1)
layout.addSpacing(10)
layout.addLayout(layout002,3)
self.setLayout(layout)
self.open_init()
pass
def open_init(self):
self.whole_df: pd.DataFrame = pd.DataFrame()
self.whole_current_df: pd.DataFrame = pd.DataFrame()
self.whole_ing_dialog = None
self.please_select_str: str = '---请选择---'
pass
def tabs_tabBarClicked(self,tab_i:int):
tab_name = self.tab_index_dict[tab_i]
self.reset_below_content()
self.open_tab_init(tab_name)
pass
def fill_table(self):
if not self.whole_current_df.empty:
df = self.whole_current_df.copy()
df['o_date'] = pd.to_datetime(df['time_str'])
df.sort_values(by='o_date',ascending=False,inplace=True)
self.table.setRowCount(len(df))
self.label_num.setText(f"共 {len(df)} 条")
data_list = df.loc[:,['id','time_str','title','short','stock','field','tagInfo']].values
for r,rv in enumerate(data_list):
for c,v in enumerate(rv):
item = QTableWidgetItem(str(v))
item.setTextAlignment(Qt.AlignmentFlag.AlignTop | Qt.AlignmentFlag.AlignLeft)
self.table.setItem(r,c,item)
pass
self.table.resizeColumnToContents(2)
pass
else:
self.table.setRowCount(0)
pass
def btn2_query_clicked(self):
stock = self.combo2_stock.currentText()
field = self.combo2_field.currentText()
tag = self.combo2_tag.currentText()
query_str = self.lineedit2_query.text()
where_list = []
if stock!=self.please_select_str:
where_list.append(f' \'{stock}\'=any(stock) ')
if field!=self.please_select_str:
where_list.append(f' \'{field}\'=any(field) ')
if tag!=self.please_select_str:
where_list.append(f' \'{tag}\'=any(tagInfo) ')
if len(query_str)>0:
where_list.append(f' title=\'%{query_str}%\' ')
pass
if where_list:
where_str = ' and '.join(where_list)
res = postgresql_utils.query_data_by_query(where_str)
stock_list, field_list, tagInfo_list = self.res_trans_df(res)
self.fill_table()
pass
else:
QMessageBox.information(self,'提示','筛选条件不能为空',QMessageBox.StandardButton.Ok)
pass
def btn3_update_clicked(self,tar_msg:str,a0:QPushButton):
self.progress = QProgressDialog('正在更新...', None, 0, 0, self)
# self.progress.setWindowFlags(self.progress.windowFlags() & ~Qt.WindowType.WindowCloseButtonHint)
self.progress.setWindowModality(Qt.WindowModality.WindowModal)
# self.progress.setWindowModality(Qt.WindowModality.NonModal)
self.progress.show()
self.insert_textedit3_log(f'{a0.text()} 开始更新...')
if tar_msg == 'ths7x24':
res = postgresql_utils.query_max_timestamp()
timestamp = int(res[0].timestamp())
# print(timestamp)
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36'
}
sql_str = '''
insert into t_ths7x24msg (time_obj,id,seq,title,digest,url,tag,ctime,rtime,source,picUrl,short,stock,field,tagInfo,nature,import,time_str) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);
'''
res_data_list = []
is_break = False
for i in range(1, 100):
if is_break:
break
url = f'https://news.10jqka.com.cn/tapp/news/push/stock/?page={i}&tag=&track=website&pagesize=400'
response = requests.get(url, headers=header)
if response.status_code == 200:
json_obj = json.loads(response.text)
one_list = json_obj['data']['list']
# print('one_list ',len(one_list))
for one in one_list:
ctime = one['ctime']
if int(ctime)<timestamp:
is_break = True
break
time_str = datetime.fromtimestamp(int(ctime)).strftime('%Y-%m-%d %H:%M:%S')
stock = []
for s in one['stock']:
stock.append(s['name'])
field = []
for f in one['field']:
field.append(f['name'])
tagInfo = []
for t in one['tagInfo']:
tagInfo.append(t['name'])
res_data_list.append((
time_str,
one['id'],
one['seq'],
one['title'],
one['digest'],
one['url'],
one['tag'],
one['ctime'],
one['rtime'],
one['source'],
one['picUrl'],
one['short'],
stock,
field,
tagInfo,
one['nature'],
one['import'],
time_str
))
pass
pass
else:
print(f'requests失败，{response.status_code}')
# print(i)
pass
conn = postgresql_utils.connect_db()
cur = conn.cursor()
# print('res_data_list ',len(res_data_list))
try:
cur.executemany(sql_str, res_data_list)
conn.commit()
except:
traceback.print_exc()
conn.rollback()
finally:
cur.close()
conn.close()
elif tar_msg == 'ths_gdqh':
# 滚动期货 https://goodsfu.10jqka.com.cn/qhgd_list/index_1.shtml
url = 'https://goodsfu.10jqka.com.cn/qhgd_list/'
belong_type = '滚动期货'
self.get_data_from_html(url,belong_type)
pass
elif tar_msg == 'ths_qsyw':
# 期市要闻
url = 'http://goodsfu.10jqka.com.cn/futuresnews_list/'
belong_type = '期市要闻'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_spqh':
# 商品期货
url = 'http://goodsfu.10jqka.com.cn/spqh_list/'
belong_type = '商品期货'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_xhsc':
# 现货市场
url = 'http://goodsfu.10jqka.com.cn/xhsc_list/'
belong_type = '现货市场'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_cjyw':
# 财经要闻
url = 'https://news.10jqka.com.cn/today_list/'
belong_type = '财经要闻'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_hgjj':
# 宏观经济
url = 'http://news.10jqka.com.cn/cjzx_list/'
belong_type = '宏观经济'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_cjxw':
# 产经新闻
url = 'http://news.10jqka.com.cn/cjkx_list/'
belong_type = '产经新闻'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_gjcj':
# 国际财经
url = 'http://news.10jqka.com.cn/guojicj_list/'
belong_type = '国际财经'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_jrsc':
# 金融市场
url = 'http://news.10jqka.com.cn/jrsc_list/'
belong_type = '金融市场'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_gsxw':
# 公司新闻
url = 'http://news.10jqka.com.cn/fssgsxw_list/'
belong_type = '公司新闻'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_qyjj':
# 区域经济
url = 'http://news.10jqka.com.cn/region_list/'
belong_type = '区域经济'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_cjpl':
# 财经评论
url = 'http://news.10jqka.com.cn/fortune_list/'
belong_type = '财经评论'
self.get_data_from_html(url, belong_type)
pass
elif tar_msg == 'ths_cjrw':
# 财经人物
url = 'http://news.10jqka.com.cn/cjrw_list/'
belong_type = '财经人物'
self.get_data_from_html(url, belong_type)
pass
else:
QMessageBox.information(self,'提示',f"{tar_msg} 无更新业务",QMessageBox.StandardButton.Ok)
return
self.progress.cancel()
self.insert_textedit3_log(f'{a0.text()} 更新到当前最新')
QMessageBox.information(self,'提示',f'{a0.text()} 更新到当前最新',QMessageBox.StandardButton.Ok)
pass
def btn1_query_clicked(self):
self.lineedit000_title.clear()
self.textedit000_content.clear()
self.lineedit002_url.clear()
stock = self.combo_stock.currentText()
field = self.combo_field.currentText()
tag = self.combo_tag.currentText()
df = self.whole_df.copy()
df = df.loc[(df['stock'].str.contains(stock) if stock!=self.please_select_str else True) & (df['field'].str.contains(field) if field!=self.please_select_str else True) & (df['tagInfo'].str.contains(tag) if tag!=self.please_select_str else True)]
self.whole_current_df = df.copy()
self.fill_table()
pass
def btn1_reset_clicked(self):
self.combo_stock.setCurrentText(self.please_select_str)
self.combo_field.setCurrentText(self.please_select_str)
self.combo_tag.setCurrentText(self.please_select_str)
self.lineedit000_title.clear()
self.textedit000_content.clear()
self.lineedit002_url.clear()
self.whole_current_df = self.whole_df.copy()
self.fill_table()
pass
def table_itemClicked(self,cur_item):
cur_row = cur_item.row()
id = self.table.item(cur_row, 0).text()
df = self.whole_current_df.copy()
df = df.loc[df['id'] == id]
if df.empty:
QMessageBox.information(self,'提示','数据不存在',QMessageBox.StandardButton.Ok)
else:
self.lineedit000_title.setText(df.iloc[0]['title'])
self.textedit000_content.setText(df.iloc[0]['digest'])
self.lineedit002_url.setText(df.iloc[0]['url'])
pass
pass
def dateedit_userDateChanged(self,a0:QDate):
date_str = a0.toString('yyyy-MM-dd')
self.label_title.setText(date_str)
self.fill_data_tab1()
pass
def fill_data_tab1(self):
self.combo_stock.clear()
self.combo_field.clear()
self.combo_tag.clear()
date_str = self.dateedit.dateTime().toString('yyyy-MM-dd')
self.progress = QProgressDialog(f'正在查询{date_str}的数据...', None, 0, 0, self)
self.progress.setWindowFlags(self.progress.windowFlags() & ~Qt.WindowType.WindowCloseButtonHint)
self.progress.setWindowModality(Qt.WindowModality.WindowModal)
# self.progress.setWindowModality(Qt.WindowModality.NonModal)
self.progress.show()
left = f'{date_str} 00:00:00'
right = f'{date_str} 23:59:59'
# id,time_str,title,short,digest,url,stock,field,tagInfo
res = postgresql_utils.query_date_by_duration(left, right)
stock_list,field_list,tagInfo_list = self.res_trans_df(res)
stock_list = list(set(stock_list))
field_list = list(set(field_list))
tagInfo_list = list(set(tagInfo_list))
self.combo_stock.addItem(self.please_select_str)
self.combo_field.addItem(self.please_select_str)
self.combo_tag.addItem(self.please_select_str)
self.combo_stock.addItems(stock_list)
self.combo_field.addItems(field_list)
self.combo_tag.addItems(tagInfo_list)
self.fill_table()
self.progress.cancel()
pass
def fill_data_tab2(self):
self.combo2_stock.clear()
self.combo2_field.clear()
self.combo2_tag.clear()
self.lineedit2_query.clear()
res = postgresql_utils.query_filters_in_all()
stock_list = []
field_list = []
tagInfo_list = []
for item in res:
stock_list.extend(item[0])
field_list.extend(item[1])
tagInfo_list.extend(item[2])
stock_list = list(set(stock_list))
field_list = list(set(field_list))
tagInfo_list = list(set(tagInfo_list))
self.combo2_stock.addItem(self.please_select_str)
self.combo2_field.addItem(self.please_select_str)
self.combo2_tag.addItem(self.please_select_str)
self.combo2_stock.addItems(stock_list)
self.combo2_field.addItems(field_list)
self.combo2_tag.addItems(tagInfo_list)
pass
def get_data_from_html(self,url_str:str,belong_type:str):
res = postgresql_utils.query_max_timestamp_in_one_belongtype(belong_type)
time_str = res[0].strftime('%Y-%m-%d %H:%M:%S')
res_hashs00 = postgresql_utils.query_hashcode_in_maxtimestamp(time_str,belong_type)
res_hashs = []
for item in res_hashs00[0]:
res_hashs.append(item)
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36'
}
res_data_list = []
is_break = False
try:
for i in range(1,21):
if is_break:
break
target_url = f"{url_str}index_{i}.shtml"
response = requests.get(target_url,headers=header)
if response.status_code == 200:
soup = BeautifulSoup(response.text, 'lxml')
elements = soup.select('.list-con')
list_con = elements[0]
li_list = list_con.find_all('li')
data_list = []
for li in li_list:
a = li.find('span').find('a')
url = a.get('href')
title = a.get('title')
span = li.find('span').find('span')
time_str = span.string
# 信息的月份与 实际时间月份比较，如果实际时间月份比信息月份小，说明实际时间已经进入下一年，信息是上一年的信息
month_str = time_str.split('月')[0]
now_month = datetime.now().month
if now_month<int(month_str):
year_int = datetime.now().year - 1
else:
year_int = datetime.now().year
timestamp = datetime.strptime(str(year_int) + '年' + time_str, '%Y年%m月%d日 %H:%M').timestamp()
timestamp_str = str(int(timestamp))
time_str00 = datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
pre_hash_str = f"{timestamp_str}{title}".encode('utf-8')
hash_obj = hashlib.sha256(pre_hash_str)
hash_val = hash_obj.hexdigest()
# print(hash_val)
if hash_val in res_hashs:
is_break = True
break
data_list.append([
belong_type,
hash_val,
time_str00,
timestamp_str,
title,
url,
time_str00
])
res_data_list.extend(data_list)
pass
else:
print(f"{target_url},请求返回码 {response.status_code}")
except:
traceback.print_exc()
self.insert_textedit3_log(traceback.format_exc())
pass
sql_str = '''
insert into t_thsfinancemsg (belong_type,hash_code,time_obj,timestamp_str,title,url,time_str) values (%s,%s,%s,%s,%s,%s,%s);
'''
conn = postgresql_utils.connect_db()
cur = conn.cursor()
try:
cur.executemany(sql_str, res_data_list)
conn.commit()
pass
except:
traceback.print_exc()
conn.rollback()
pass
finally:
cur.close()
conn.close()
pass
def res_trans_df(self,res)->tuple:
data_list = []
stock_list = []
field_list = []
tagInfo_list = []
for item in res:
stock_list.extend(item[6])
field_list.extend(item[7])
tagInfo_list.extend(item[8])
data_list.append([
item[0], # id
item[1], # time_str
item[2], # title
item[3], # short
item[4], # digest
item[5], # url
','.join(item[6]), # stock
','.join(item[7]), # field
','.join(item[8]) # tagInfo
])
df = pd.DataFrame(columns=['id', 'time_str', 'title', 'short', 'digest', 'url', 'stock', 'field', 'tagInfo'],
data=data_list)
self.whole_df = df.copy()
self.whole_current_df = df.copy()
return stock_list,field_list,tagInfo_list
def reset_below_content(self):
self.table.setRowCount(0)
self.lineedit000_title.clear()
self.textedit000_content.clear()
self.lineedit002_url.clear()
pass
def open_tab_init(self,tab_name:str):
if tab_name == self.ONEDAY_TAB:
res = postgresql_utils.query_duration_timestamp()
self.dateedit.setDate(res[1])
self.label_duration.setText(f"从 {str(res[0])} 到 {res[1]}")
self.fill_data_tab1()
pass
elif tab_name == self.QUERY_TAB:
self.fill_data_tab2()
pass
elif tab_name == self.UPDATE_TAB:
pass
else:
QMessageBox.information(self,'提示',f'{tab_name} 不存在',QMessageBox.StandardButton.Ok)
pass
def insert_textedit3_log(self,msg:str):
self.textedit3_log.append(msg+'\n')
pass