npu-model-upload
into V20221102
1 year ago
@@ -367,51 +367,58 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative | |||||
if !strings.HasSuffix(input.Prefix, "/") { | if !strings.HasSuffix(input.Prefix, "/") { | ||||
input.Prefix += "/" | input.Prefix += "/" | ||||
} | } | ||||
output, err := ObsCli.ListObjects(input) | |||||
fileInfos := make([]FileInfo, 0) | fileInfos := make([]FileInfo, 0) | ||||
prefixLen := len(input.Prefix) | prefixLen := len(input.Prefix) | ||||
fileMap := make(map[string]bool, 0) | fileMap := make(map[string]bool, 0) | ||||
if err == nil { | |||||
for _, val := range output.Contents { | |||||
log.Info("val key=" + val.Key) | |||||
var isDir bool | |||||
var fileName string | |||||
if val.Key == input.Prefix { | |||||
continue | |||||
} | |||||
fileName = val.Key[prefixLen:] | |||||
log.Info("fileName =" + fileName) | |||||
files := strings.Split(fileName, "/") | |||||
if fileMap[files[0]] { | |||||
continue | |||||
} else { | |||||
fileMap[files[0]] = true | |||||
index := 1 | |||||
for { | |||||
output, err := ObsCli.ListObjects(input) | |||||
if err == nil { | |||||
log.Info("Page:%d\n", index) | |||||
index++ | |||||
for _, val := range output.Contents { | |||||
var isDir bool | |||||
var fileName string | |||||
if val.Key == input.Prefix { | |||||
continue | |||||
} | |||||
fileName = val.Key[prefixLen:] | |||||
files := strings.Split(fileName, "/") | |||||
if fileMap[files[0]] { | |||||
continue | |||||
} else { | |||||
fileMap[files[0]] = true | |||||
} | |||||
ParenDir := relativePath | |||||
fileName = files[0] | |||||
if len(files) > 1 { | |||||
isDir = true | |||||
ParenDir += fileName + "/" | |||||
} else { | |||||
isDir = false | |||||
} | |||||
fileInfo := FileInfo{ | |||||
ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), | |||||
FileName: fileName, | |||||
Size: val.Size, | |||||
IsDir: isDir, | |||||
ParenDir: ParenDir, | |||||
} | |||||
fileInfos = append(fileInfos, fileInfo) | |||||
} | } | ||||
ParenDir := relativePath | |||||
fileName = files[0] | |||||
if len(files) > 1 { | |||||
isDir = true | |||||
ParenDir += fileName + "/" | |||||
if output.IsTruncated { | |||||
input.Marker = output.NextMarker | |||||
} else { | } else { | ||||
isDir = false | |||||
break | |||||
} | } | ||||
fileInfo := FileInfo{ | |||||
ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), | |||||
FileName: fileName, | |||||
Size: val.Size, | |||||
IsDir: isDir, | |||||
ParenDir: ParenDir, | |||||
} else { | |||||
if obsError, ok := err.(obs.ObsError); ok { | |||||
log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) | |||||
} | } | ||||
fileInfos = append(fileInfos, fileInfo) | |||||
} | |||||
return fileInfos, err | |||||
} else { | |||||
if obsError, ok := err.(obs.ObsError); ok { | |||||
log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) | |||||
return nil, err | |||||
} | } | ||||
return nil, err | |||||
} | } | ||||
return fileInfos, nil | |||||
} | } | ||||
func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { | func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { | ||||
@@ -470,7 +470,11 @@ func ModelList(ctx *context.APIContext) { | |||||
status := models.StorageScheduleSucceed | status := models.StorageScheduleSucceed | ||||
var fileInfos []storage.FileInfo | var fileInfos []storage.FileInfo | ||||
if task.ComputeResource == models.NPUResource { | if task.ComputeResource == models.NPUResource { | ||||
fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName) | |||||
prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName), "/") | |||||
if !strings.HasSuffix(prefix, "/") { | |||||
prefix += "/" | |||||
} | |||||
fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) | |||||
if err != nil { | if err != nil { | ||||
log.Info("get TrainJobListModel failed:", err) | log.Info("get TrainJobListModel failed:", err) | ||||
ctx.ServerError("GetObsListObject:", err) | ctx.ServerError("GetObsListObject:", err) | ||||
@@ -484,6 +488,10 @@ func ModelList(ctx *context.APIContext) { | |||||
if models.IsTrainJobTerminal(task.Status) { | if models.IsTrainJobTerminal(task.Status) { | ||||
if task.Status == models.GrampusStatusStopped { | if task.Status == models.GrampusStatusStopped { | ||||
status = models.StorageNoFile | status = models.StorageNoFile | ||||
} else if task.Status == models.GrampusStatusFailed { | |||||
if task.AiCenter == "" { | |||||
status = models.StorageNoFile | |||||
} | |||||
} else { | } else { | ||||
record, _ := models.GetScheduleRecordByCloudbrainID(task.ID) | record, _ := models.GetScheduleRecordByCloudbrainID(task.ID) | ||||
if record != nil { | if record != nil { | ||||
Dear OpenI User
Thank you for your continuous support to the Openl Qizhi Community AI Collaboration Platform. In order to protect your usage rights and ensure network security, we updated the Openl Qizhi Community AI Collaboration Platform Usage Agreement in January 2024. The updated agreement specifies that users are prohibited from using intranet penetration tools. After you click "Agree and continue", you can continue to use our services. Thank you for your cooperation and understanding.
For more agreement content, please refer to the《Openl Qizhi Community AI Collaboration Platform Usage Agreement》