|
- import os
- import wget
- import tarfile
- import argparse
- import subprocess
-
- from data_opts import add_data_opts
- from tqdm import tqdm
- import shutil
- # from downloader import download
-
- from utils import create_manifest
-
- # import os
- # os.environ['http_proxy'] = 'http://127.0.0.1:1089'
- # os.environ['https_proxy'] = 'https://127.0.0.1:1089'
-
- parser = argparse.ArgumentParser(description='Processes and downloads LibriSpeech dataset.')
- parser = add_data_opts(parser)
- parser.add_argument("--target-dir", default='LibriSpeech_dataset/', type=str, help="Directory to store the dataset.")
- parser.add_argument('--files-to-use', default="test-other.tar.gz", type=str,
- help='list of file names to download')
- args = parser.parse_args()
-
- LIBRI_SPEECH_URLS = {
- "test_other": ["http://www.openslr.org/resources/12/test-other.tar.gz"]
- }
-
-
- def _preprocess_transcript(phrase):
- return phrase.strip().upper()
-
-
- def _process_file(wav_dir, txt_dir, base_filename, root_dir):
- full_recording_path = os.path.join(root_dir, base_filename)
- full_recording_path = "C:\\Users\\admin\\Desktop\\test\\mindspore_jasper\\" + full_recording_path
-
- # assert os.path.exists(full_recording_path) and os.path.exists(root_dir)
- wav_recording_path = os.path.join(wav_dir, base_filename.replace(".flac", ".wav"))
- wav_recording_path = "C:\\Users\\admin\\Desktop\\test\\mindspore_jasper\\" + wav_recording_path
-
- subprocess.call(["sox {} -rw {} -b 16 -c 1 {}".format(full_recording_path, str(args.sample_rate),
- wav_recording_path)], shell=True)
-
-
- # process transcript
- txt_transcript_path = os.path.join(txt_dir, base_filename.replace(".flac", ".txt"))
- txt_transcript_path = "C:\\Users\\admin\\Desktop\\test\\mindspore_jasper\\" + txt_transcript_path
- transcript_file = os.path.join(root_dir, "-".join(base_filename.split('-')[:-1]) + ".trans.txt")
- transcript_file = "C:\\Users\\admin\\Desktop\\test\\mindspore_jasper\\" + transcript_file
-
-
- # assert os.path.exists(transcript_file), "Transcript file {} does not exist.".format(transcript_file)
-
- transcriptions = open(transcript_file).read().strip().split("\n")
-
- transcriptions = {t.split()[0].split("-")[-1]: " ".join(t.split()[1:]) for t in transcriptions}
- with open(txt_transcript_path, "w") as f:
- key = base_filename.replace(".flac", "").split("-")[-1]
- assert key in transcriptions, "{} is not in the transcriptions".format(key)
- f.write(_preprocess_transcript(transcriptions[key]))
- f.flush()
-
-
- def main():
- target_dl_dir = args.target_dir
- if not os.path.exists(target_dl_dir):
- os.makedirs(target_dl_dir)
- files_to_dl = args.files_to_use.strip().split(',')
- for split_type, lst_libri_urls in LIBRI_SPEECH_URLS.items():
- split_dir = os.path.join(target_dl_dir, split_type)
- if not os.path.exists(split_dir):
- os.makedirs(split_dir)
- split_wav_dir = os.path.join(split_dir, "wav")
- if not os.path.exists(split_wav_dir):
- os.makedirs(split_wav_dir)
- split_txt_dir = os.path.join(split_dir, "txt")
- if not os.path.exists(split_txt_dir):
- os.makedirs(split_txt_dir)
- extracted_dir = os.path.join(split_dir, "LibriSpeech")
- # if os.path.exists(extracted_dir):
- # shutil.rmtree(extracted_dir)
- for url in lst_libri_urls:
- # check if we want to dl this file
- dl_flag = False
- for f in files_to_dl:
- if url.find(f) != -1:
- dl_flag = True
- if not dl_flag:
- print("Skipping url: {}".format(url))
- continue
- filename = url.split("/")[-1]
- target_filename = os.path.join(split_dir, filename)
- if not os.path.exists(target_filename):
- pass
- # wget.download(url, split_dir)
- # download(url=url, target=split_dir, blocks=8)
- # print(target_filename)
- # print("Unpacking {}...".format(filename))
- # tar = tarfile.open(target_filename)
- # tar.extractall(split_dir)
- # tar.close()
-
- # os.remove(target_filename)
- print("Converting flac files to wav and extracting transcripts...")
- assert os.path.exists(extracted_dir), "Archive {} was not properly uncompressed.".format(filename)
- for root, subdirs, files in tqdm(os.walk(extracted_dir)):
- for f in files:
- if f.find(".flac") != -1:
- _process_file(wav_dir=split_wav_dir, txt_dir=split_txt_dir,
- base_filename=f, root_dir=root)
-
- print("Finished {}".format(url))
- shutil.rmtree(extracted_dir)
- if split_type == 'train': # Prune to min/max duration
- create_manifest(data_path=split_dir,
- output_name='libri_' + split_type + '_manifest.csv',
- manifest_path=args.manifest_dir,
- min_duration=args.min_duration,
- max_duration=args.max_duration)
- else:
- create_manifest(data_path=split_dir,
- output_name='libri_' + split_type + '_manifest.csv',
- manifest_path=args.manifest_dir)
-
-
- if __name__ == "__main__":
- main()
|