|
- import mindspore as ms
- import mindspore.dataset as ds
- import mindspore.dataset.vision.c_transforms as vision
- from mindspore.mindrecord import ImageNetToMR
- import mindspore.dataset.vision.c_transforms as C
- import mindspore.dataset.transforms.c_transforms as C2
- import mindspore.common.dtype as mstype
- import multiprocessing
- import mindspore.dataset as ds
- from mindspore.communication.management import init, get_rank, get_group_size
- import random
- import os
- import time
- import multiprocessing as mp
- import numpy as np
- import cv2
-
- import datetime
- import glob
-
- import mindspore.nn as nn
- from mindspore import context
- from mindspore.train.train_thor import ConvertModelUtils
- from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor
- from mindspore.communication.management import init, get_rank
- from mindspore.parallel import set_algo_parameters
-
- import mindspore.log as logger
- import moxing as mox
- from src.lr_generator import get_lr, warmup_cosine_annealing_lr
- from src.CrossEntropySmooth import CrossEntropySmooth
- from src.eval_callback import EvalCallBack
- from src.metric import DistAccuracy, ClassifyCorrectCell
- from src.model_utils.config import config
- from src.model_utils.moxing_adapter import moxing_wrapper
- from src.model_utils.device_adapter import get_rank_id, get_device_num
- from src.resnet import conv_variance_scaling_initializer
-
-
-
- def get_train_list(data_dir):
- print("******************job start flag***************")
- g = os.walk(data_dir)
- path_list = []
- lable = 0
- file = open('/home/work/user-job-dir/data/imagenet/train/train.txt','w')
- for path, dir_list, file_list in g:
- for i in dir_list:
- img_path = i + " " + str(lable) + '\n'
- file.write(img_path)
- lable = lable+1
-
-
-
- if __name__ == '__main__':
-
-
- #将数据集从local拷贝到推理镜像中:
- local_data_url = config.data_url
- print(os.path.exists(local_data_url))
- config.data_url = '/home/work/user-job-dir/data/'
- try:
- mox.file.copy_parallel(local_data_url, config.data_url)
- print("Successfully Download {} to {}".format(local_data_url,config.data_url))
- except Exception as e:
- print('moxing download {} to {} failed: '.format(local_data_url, config.data_url) + str(e))
-
-
-
- local_train_url = config.train_url
- config.train_url = '/home/work/user-job-dir/model/'
- if not os.path.exists(config.train_url):
- try:
- os.mkdir(config.train_url)
- except:
- print(config.train_url, "already exist !")
-
- data_dir = '/home/work/user-job-dir/data/imagenet/train/'
-
-
- data = get_train_list(data_dir)
- record_save_dir = config.train_url + "MindRecord/"
-
- if not os.path.exists(record_save_dir):
- try:
- os.mkdir(record_save_dir)
- except:
- print(record_save_dir, "already exist !")
-
-
- IMAGENET_MAP_FILE = "/home/work/user-job-dir/data/imagenet/train/train.txt"
- IMAGENET_IMAGE_DIR = "/home/work/user-job-dir/data/imagenet/train"
- MINDRECORD_FILE = record_save_dir + "imagenet_tiny_train.mindrecord"
- imagenet_transformer = ImageNetToMR(IMAGENET_MAP_FILE, IMAGENET_IMAGE_DIR, MINDRECORD_FILE, partition_number=1)
- imagenet_transformer.transform()
- print("*****************job done flag***********************")
-
- try:
- mox.file.copy_parallel(config.train_url, local_train_url)
- print("Successfully Upload {} to {}".format(config.train_url, local_train_url))
- except Exception as e:
- print('moxing upload {} to {} failed: '.format(config.train_url, local_train_url) + str(e))
|