python
复制代码
import os
import json
import random
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
def read_json_file(file_path):
try:
with open(file_path, 'r', encoding='utf-8') as file:
data = json.load(file)
return data
except FileNotFoundError:
print(f"File {file_path} not found.")
except json.JSONDecodeError:
print(f"File {file_path} is not a valid JSON file.")
except Exception as e:
print(f"An error occurred: {e}")
def read_jsonl_file(file_path):
data = []
with open(file_path, 'r', encoding='utf-8') as file:
for line in file:
try:
data.append(json.loads(line))
except:
print(line)
1/0
return data
def read_praquet_file(file_path):
table = pq.read_table(file_path)
df = table.to_pandas()
result=[row.tolist() for _, row in df.iterrows()]
return result
def save_json(file_path,data):
with open(file_path, 'w', encoding='utf-8') as file:
json.dump(data, file, indent=4, ensure_ascii=False)
print(f'Save {file_path} is ok!')
def save_jsonl(file_path,data):
try:
with open(file_path, 'w', encoding='utf-8') as file:
for item in data:
file.write(json.dumps(item, ensure_ascii=False) + '\n')
print(f"Data saved to {file_path}")
except Exception as e:
print(f"An error occurred while saving the data: {e}")
def save_parquet(file_path, data):
if isinstance(data, list):
data = pd.DataFrame(data)
if not isinstance(data, pd.DataFrame):
raise ValueError("data must be a pandas DataFrame or a list of lists")
pq.write_table(pa.Table.from_pandas(data), file_path)
print(f'Save {file_path} is ok!')
def convert_lists_to_json(df):
"""Convert lists in DataFrame to JSON strings."""
for column in df.columns:
if df[column].apply(lambda x: isinstance(x, list)).any():
df[column] = df[column].apply(lambda x: json.dumps(x) if isinstance(x, list) else x)
return df
root='/path/to/parquet/dir'
save_path='/path/to/savedir/save_name.json'
new_data=[]
dirs=os.listdir(root)
for one in dirs:
if one.endswith('.parquet'):
print(one)
file_path=root+'/'+one
data=read_praquet_file(file_path)
for x in data:
inp=x[3]
res=x[4]
new_entry={
"conversations": [
{
"role": "user",
"content": inp
},
{
"role": "assistant",
"content": res
}
]
}
if len(inp)>0 and len(res)>0:
new_data+=[new_entry]
save_json(save_path,new_data)