|
- # 用于单独生成验证集 features
-
- import os
- import numpy as np
-
- import time
- import h5py
- import pandas as pd
- import pickle
-
- os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
- os.environ["CUDA_VISIBLE_DEVICES"] = "0"
-
- #=========================================================================================================#
- data_path = '/dataset'
- train_csv = data_path + '/train_origin_spec.csv'
- val_csv = data_path + '/fold1_evaluate.csv'
-
- # logmel 特征位置
- feat_path = data_path + '/features'
-
- # 保存的已计算的 deltas 特征位置
- feature_path = '/model/'
-
- delta = True
-
- #=========================================================================================================#
- # 加载 logmel 文件,获取特征和标签
- def load_data_2020(feat_path, csv_path):
- with open(csv_path, 'r') as text_file:
- lines = text_file.read().split('\n')
- for idx, elem in enumerate(lines):
- lines[idx] = lines[idx].split('\t')
- lines[idx][0] = lines[idx][0].split('/')[-1].split('.')[0]
-
- lines = lines[1:]
- lines = [elem for elem in lines if elem != ['']]
- for idx, elem in enumerate(lines):
- lines[idx][-1] = lines[idx][-1].split('\r')[0]
- label_info = np.array(lines)
-
- data_df = pd.read_csv(csv_path, sep='\t', encoding='ASCII')
- labels = data_df['scene_label'].astype('category').cat.codes.values
-
- feat_mtx = []
- for [filename, labnel] in label_info:
- filepath = feat_path + '/' + filename + '.logmel'
- with open(filepath,'rb') as f:
- temp=pickle.load(f, encoding='latin1')
- feat_mtx.append(temp['feat_data'])
-
- feat_mtx = np.array(feat_mtx)
- return feat_mtx, labels
-
- # calculate_deltas
- def deltas(X_in):
- X_out = (X_in[:,:,2:,:]-X_in[:,:,:-2,:])/10.0
- X_out = X_out[:,:,1:-1,:]+(X_in[:,:,4:,:]-X_in[:,:,:-4,:])/5.0
- return X_out
-
- # calculate_deltas
- def calculate_deltas(LM_feature):
- LM_deltas_data = deltas(LM_feature)
- LM_deltas_deltas_data = deltas(LM_deltas_data)
- LM_feature = np.concatenate((LM_feature[:,:,4:-4,:],LM_deltas_data[:,:,2:-2,:],LM_deltas_deltas_data),axis=-1)
-
- return LM_feature
-
- # 保存特征
- def write_hdf5(feature_path, y_train, y_val, data_train, data_val):
- # Hdf5 file for storing features and targets
- hf = h5py.File(feature_path, 'w')
- # labels
- hf.create_dataset(
- name='y_train',
- data=y_train,)
- hf.create_dataset(
- name='y_val',
- data=y_val,)
- # features
- hf.create_dataset(
- name='data_train',
- data=data_train,
- dtype=np.float32)
- hf.create_dataset(
- name='data_val',
- data=data_val,
- dtype=np.float32)
-
- hf.close()
-
- # 载入特征
- def load_hdf5(hdf5_path):
- '''Load hdf5 file.
-
- Returns:
- data_dict: dict of data, e.g.:
- {'audio_name': np.array(['a.wav', 'b.wav', ...]),
- 'feature': (audios_num, frames_num, mel_bins)
- 'target': (audios_num,),
- ...}
- '''
- data_dict = {}
-
- with h5py.File(hdf5_path, 'r') as hf:
- data_dict['y_train'] = hf['y_train'][:]
- data_dict['y_val'] = hf['y_val'][:]
- data_dict['data_train'] = hf['data_train'][:].astype(np.float32)
- data_dict['data_val'] = hf['data_val'][:].astype(np.float32)
-
- return data_dict
-
- #=========================================================================================================#
- # 读取 logmel 文件,获取特征
- extract_time = time.time()
- data_val, y_val = load_data_2020(feat_path, val_csv)
- # data_train, y_train = load_data_2020(feat_path, train_csv)
- data_train = ''
- y_train = ''
- print('Number of audios: {}'.format(len(data_train)))
-
- # 训练集特征
- # print('training data dimension: ')
- # if delta:
- # # data_train = calculate_deltas(data_train)
- # print(data_train.shape)
-
- # 验证集特征
- print('validation data dimension: ')
- if delta:
- data_val = calculate_deltas(data_val)
- print(data_val.shape)
-
- print('Extract Mel Features Time spent: {} s'.format(time.time() - extract_time))
-
-
- # 保存特征
- print("Save feature:")
- save_time = time.time()
- # 保存的特征文件名
- feature_path = feature_path + 'feature_val' + '.h5'
- # Hdf5 file for storing features and targets
- write_hdf5(feature_path, y_train, y_val, data_train, data_val)
- print('Write hdf5 file to {} using {:.3f} s'.format(feature_path, time.time() - save_time))
|