services/ai_task_service/cluster/c2net.go

@@ -457,7 +457,7 @@ func generateGrampusTrainCommand(req entity.CreateTrainTaskRequest) string {
 		//export
 		Add(buildExportCommand(req.Name, computeResource)).
 		//exec code
-		Add(buildExecCodeCommand(path.Join(codePath, strings.ToLower(t.RepoName)), modelFilePath, t.BootFile, computeResource, req.Name, t.Params))
+		Add(buildExecCodeCommand(path.Join(codePath, strings.ToLower(t.RepoName)), modelFilePath, t.BootFile, computeResource, req.Name, t.Params, t.Datasets, datasetPath))
 
 	return builder.ToString()
 }
@@ -526,6 +526,49 @@ func buildUnzipDatasetCommand(datasets []entity.ContainerData, datasetPath, comp
 	return builder
 }
 
+func buildDeleteUnzipDatasetCommand(builder *entity.CommandBuilder, datasets []entity.ContainerData, datasetPath, computeSource string) {
+	if computeSource == models.NPU {
+		return
+	}
+	if len(datasets) == 0 {
+		return
+	}
+	builder.Next(entity.NewCommand("cd", datasetPath)).
+		Next(entity.NewCommand("echo", "'start to delete unzip datasets'"))
+
+	fileDatasets := make([]entity.ContainerData, 0)
+	for _, dataset := range datasets {
+		if !dataset.IsDir {
+			fileDatasets = append(fileDatasets, dataset)
+		}
+	}
+	//单数据集
+	if len(fileDatasets) == 1 {
+
+		builder.Next(entity.NewCommand("find . ! -name", "'"+fileDatasets[0].Name+"'", "-type f -exec rm -f {} +"))
+		builder.Next(entity.NewCommand("find . -type d ! -name", "'"+fileDatasets[0].Name+"'", "-and ! -name . -and ! -name ..  -exec rm -rf {} +"))
+
+	} else {
+		//多数据集
+		for i := 0; i < len(fileDatasets); i++ {
+
+			builder.Next(entity.NewCommand("rm", "-rf", "'"+getZipFileNameExcludeExt(fileDatasets[i].Name)+"'"))
+
+		}
+	}
+	builder.Next(entity.NewCommand("ls", "-l"))
+	builder.Next(entity.NewCommand("echo", "'delete unzip datasets finished'"))
+}
+
+func getZipFileNameExcludeExt(fileName string) string {
+	if strings.HasSuffix(fileName, ".tar.gz") {
+		return fileName[0 : len(fileName)-7]
+	} else if strings.HasSuffix(fileName, ".zip") {
+		return fileName[0 : len(fileName)-4]
+	}
+	return fileName
+}
+
 func buildExportCommand(jobName, computeResource string) *entity.CommandBuilder {
 	builder := &entity.CommandBuilder{}
 
@@ -539,7 +582,7 @@ func buildExportCommand(jobName, computeResource string) *entity.CommandBuilder
 	return builder
 }
 
-func buildExecCodeCommand(codeDirPath, modelFilePath, bootFile, computeResource, jobName string, params models.Parameters) *entity.CommandBuilder {
+func buildExecCodeCommand(codeDirPath, modelFilePath, bootFile, computeResource, jobName string, params models.Parameters, datasets []entity.ContainerData, datasetPath string) *entity.CommandBuilder {
 	builder := &entity.CommandBuilder{}
 	builder.Next(entity.NewCommand("echo", "'start to exec code'"))
 
@@ -575,6 +618,9 @@ func buildExecCodeCommand(codeDirPath, modelFilePath, bootFile, computeResource,
 	}
 
 	builder.Next(entity.NewCommand("result=$?"))
+	//delete unzip dataset
+	buildDeleteUnzipDatasetCommand(builder, datasets, datasetPath, computeResource)
+
 	builder.Next(entity.NewCommand("bash", "-c", "\"[[ $result -eq 0 ]] && exit 0 || exit -1\""))
 	return builder
 }