From 39b6d649d90234ec3e80e6fd90a04125aa6abd67 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 15 Dec 2023 15:17:11 +0800 Subject: [PATCH 1/2] clear enviroment --- services/ai_task_service/cluster/c2net.go | 50 ++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/services/ai_task_service/cluster/c2net.go b/services/ai_task_service/cluster/c2net.go index 3ac0679a1d..c5eeb57cd5 100644 --- a/services/ai_task_service/cluster/c2net.go +++ b/services/ai_task_service/cluster/c2net.go @@ -457,8 +457,9 @@ func generateGrampusTrainCommand(req entity.CreateTrainTaskRequest) string { //export Add(buildExportCommand(req.Name, computeResource)). //exec code - Add(buildExecCodeCommand(path.Join(codePath, strings.ToLower(t.RepoName)), modelFilePath, t.BootFile, computeResource, req.Name, t.Params)) - + Add(buildExecCodeCommand(path.Join(codePath, strings.ToLower(t.RepoName)), modelFilePath, t.BootFile, computeResource, req.Name, t.Params)). + //delete unzip dataset + Add(buildDeleteUnzipDatasetCommand(t.Datasets, datasetPath, computeResource)) return builder.ToString() } @@ -526,6 +527,51 @@ func buildUnzipDatasetCommand(datasets []entity.ContainerData, datasetPath, comp return builder } +func buildDeleteUnzipDatasetCommand(datasets []entity.ContainerData, datasetPath, computeSource string) *entity.CommandBuilder { + builder := &entity.CommandBuilder{} + if computeSource == models.NPU { + return builder + } + if len(datasets) == 0 { + return nil + } + builder.Next(entity.NewCommand("cd", datasetPath)). + Next(entity.NewCommand("echo", "'start to delete unzip datasets'")) + + fileDatasets := make([]entity.ContainerData, 0) + for _, dataset := range datasets { + if !dataset.IsDir { + fileDatasets = append(fileDatasets, dataset) + } + } + //单数据集 + if len(fileDatasets) == 1 { + + builder.Next(entity.NewCommand("find . ! -name", "'"+fileDatasets[0].Name+"'", "-type f -exec rm -f {} +")) + builder.Next(entity.NewCommand("find . -type d ! -name", "'"+fileDatasets[0].Name+"'", "-and ! -name '.' -and ! -name '..' -exec sudo rm -rf {} +")) + + } else { + //多数据集 + for i := 0; i < len(fileDatasets); i++ { + + builder.Next(entity.NewCommand("rm", "-rf", "'"+getZipFileNameExcludeExt(fileDatasets[i].Name)+"'")) + + } + } + builder.Next(entity.NewCommand("ls", "-l")) + builder.Next(entity.NewCommand("echo", "'delete unzip datasets finished'")) + return builder +} + +func getZipFileNameExcludeExt(fileName string) string { + if strings.HasSuffix(fileName, ".tar.gz") { + return fileName[0 : len(fileName)-7] + } else if strings.HasSuffix(fileName, ".zip") { + return fileName[0 : len(fileName)-4] + } + return fileName +} + func buildExportCommand(jobName, computeResource string) *entity.CommandBuilder { builder := &entity.CommandBuilder{} -- 2.34.1 From 937b917c0b51fe56ec6f1185ff0798b8866d691a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 18 Dec 2023 10:53:00 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/ai_task_service/cluster/c2net.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/services/ai_task_service/cluster/c2net.go b/services/ai_task_service/cluster/c2net.go index c5eeb57cd5..e9da8821f4 100644 --- a/services/ai_task_service/cluster/c2net.go +++ b/services/ai_task_service/cluster/c2net.go @@ -457,9 +457,8 @@ func generateGrampusTrainCommand(req entity.CreateTrainTaskRequest) string { //export Add(buildExportCommand(req.Name, computeResource)). //exec code - Add(buildExecCodeCommand(path.Join(codePath, strings.ToLower(t.RepoName)), modelFilePath, t.BootFile, computeResource, req.Name, t.Params)). - //delete unzip dataset - Add(buildDeleteUnzipDatasetCommand(t.Datasets, datasetPath, computeResource)) + Add(buildExecCodeCommand(path.Join(codePath, strings.ToLower(t.RepoName)), modelFilePath, t.BootFile, computeResource, req.Name, t.Params, t.Datasets, datasetPath)) + return builder.ToString() } @@ -527,13 +526,12 @@ func buildUnzipDatasetCommand(datasets []entity.ContainerData, datasetPath, comp return builder } -func buildDeleteUnzipDatasetCommand(datasets []entity.ContainerData, datasetPath, computeSource string) *entity.CommandBuilder { - builder := &entity.CommandBuilder{} +func buildDeleteUnzipDatasetCommand(builder *entity.CommandBuilder, datasets []entity.ContainerData, datasetPath, computeSource string) { if computeSource == models.NPU { - return builder + return } if len(datasets) == 0 { - return nil + return } builder.Next(entity.NewCommand("cd", datasetPath)). Next(entity.NewCommand("echo", "'start to delete unzip datasets'")) @@ -548,7 +546,7 @@ func buildDeleteUnzipDatasetCommand(datasets []entity.ContainerData, datasetPath if len(fileDatasets) == 1 { builder.Next(entity.NewCommand("find . ! -name", "'"+fileDatasets[0].Name+"'", "-type f -exec rm -f {} +")) - builder.Next(entity.NewCommand("find . -type d ! -name", "'"+fileDatasets[0].Name+"'", "-and ! -name '.' -and ! -name '..' -exec sudo rm -rf {} +")) + builder.Next(entity.NewCommand("find . -type d ! -name", "'"+fileDatasets[0].Name+"'", "-and ! -name . -and ! -name .. -exec rm -rf {} +")) } else { //多数据集 @@ -560,7 +558,6 @@ func buildDeleteUnzipDatasetCommand(datasets []entity.ContainerData, datasetPath } builder.Next(entity.NewCommand("ls", "-l")) builder.Next(entity.NewCommand("echo", "'delete unzip datasets finished'")) - return builder } func getZipFileNameExcludeExt(fileName string) string { @@ -585,7 +582,7 @@ func buildExportCommand(jobName, computeResource string) *entity.CommandBuilder return builder } -func buildExecCodeCommand(codeDirPath, modelFilePath, bootFile, computeResource, jobName string, params models.Parameters) *entity.CommandBuilder { +func buildExecCodeCommand(codeDirPath, modelFilePath, bootFile, computeResource, jobName string, params models.Parameters, datasets []entity.ContainerData, datasetPath string) *entity.CommandBuilder { builder := &entity.CommandBuilder{} builder.Next(entity.NewCommand("echo", "'start to exec code'")) @@ -621,6 +618,9 @@ func buildExecCodeCommand(codeDirPath, modelFilePath, bootFile, computeResource, } builder.Next(entity.NewCommand("result=$?")) + //delete unzip dataset + buildDeleteUnzipDatasetCommand(builder, datasets, datasetPath, computeResource) + builder.Next(entity.NewCommand("bash", "-c", "\"[[ $result -eq 0 ]] && exit 0 || exit -1\"")) return builder } -- 2.34.1