python - 快速搜索指定格式文件的方法

需求1：快速查找指定文件夹下，所有满足开头为NC，结尾为.nc的文件：

python 复制代码

root_path = "/Users/xpji/convert_kuihua9data/20230621"

from pathlib import Path

def get_filtered_file_paths(root_path):
    filtered_files = [str(file_path) for file_path in Path(root_path).rglob('NC*.nc')]
    print(filtered_files)
    return filtered_files

filtered_files = get_filtered_file_paths(root_path)

需求2：满足指定文件夹下，所有后缀为.HDF5的文件，并排序：

python 复制代码

gpm_path = r'/Datadisk/GPM/GPM_30min/2017/'
gpm_pattern = '*.HDF5'

### 方法1
# 使用列表推导式和Path对象快速获取满足条件的文件路径
gpm_list = [str(file_path) for file_path in Path(gpm_path).rglob(gpm_pattern)]
gpm_list.sort()
# 输出结果
print(gpm_list)


### 方法2
# 初始化文件路径列表
gpm_list2 = []
# 遍历所有以'001'到'031'开头的文件夹
for i in range(1, 366):
    # 构建每个文件夹的名称，如果数字小于10，则需要在前面加0
    folder_name = str(i).zfill(3)
    folder = os.path.join(gpm_path, folder_name)
    # 获取所有匹配到的文件路径
    files = glob.glob(os.path.join(folder, gpm_pattern))
    # 将匹配到的文件路径加入列表
    gpm_list2.extend(files)
gpm_list2.sort()

相比之下，第一种方法速度更快，只需要满足筛选条件后排序即可；第二种是遍历文件夹后再排序

需求3：生成指定一年的所有天数，格式为年-月-日，数据类型为date，如2017-12-31

python 复制代码

ds = []
year = 2017
for month in range(1, 13):
    days_in_month = calendar.monthrange(year, month)[1]
    
    for day in range(1, days_in_month + 1):
        date = datetime.date(year, month, day)
        ds.append(date)
        print(date)

需求4 ：匹配GPM和Sate卫星数据种相同时刻的数据：
- 方法1

python 复制代码

gpm_abspath  = [os.path.abspath(i) for i in gpm_filtered_files]
sate_abspath = [os.path.abspath(i) for i in sate_filtered_files]

# 获取 gpm_list 和 sate_list 中文件名部分
gpm_dir = [os.path.basename(i) for i in gpm_abspath]
sate_dir = [os.path.basename(i) for i in sate_abspath]

match = []
for i in tqdm(range(len(gpm_dir))):
    
    index1 = gpm_dir[i].split('.')[4][:8] + ' ' + gpm_dir[i].split('.')[4][10:14] 
    
    for j in range(len(sate_dir)):
        
        index2 =  sate_dir[j].split('_')[2]  + ' ' + sate_dir[j].split('_')[3]
        
        if index1 == index2:
            
            match.append(gpm_abspath[i])
            
            print(gpm_abspath[i])

方法2

python 复制代码

gpm_dir2 = [filepath.split('.')[4][:8] + ' ' + filepath.split('.')[4][10:14] for filepath in gpm_abspath]

sate_dict = {}
gpm_dict = {}

for filepath in sate_abspath:
    
    index = filepath.split('_')[2] + ' ' + filepath.split('_')[3]
    sate_dict[index] = filepath
    
for filepath in gpm_abspath:
    
    index = filepath.split('.')[4][:8] + ' ' + filepath.split('.')[4][10:14]
    gpm_dict[index] = filepath

match_sate_path = []
match_gpm_path = []
for index in tqdm(set(gpm_dir2) & set(sate_dict.keys())):
    match_sate_path.append(sate_dict[index])
    match_gpm_path.append(gpm_dict[index])
    print(sate_dict[index])
match_sate_path.sort()
match_gpm_path.sort()

需求5：保存路径为.pkl,并读取文件

python 复制代码

with open('match_gpm_list_2017.pkl', 'wb') as f:
    pickle.dump(match_gpm_path, f)
    print('match_gpm_list_2017.pkl','save finish')
    
with open('match_sate_list_2017.pkl', 'wb') as f:
    pickle.dump(match_sate_path, f)
    print('match_sate_list_2017.pkl','save finish')

with open('my_list.pkl', 'rb') as f:
    gpm_list = pickle.load(f)