From f878cbb81e176349717a0f4a82c472205a7a3de8 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 26 Jul 2022 16:46:36 +0800 Subject: [PATCH 1/4] modelarts restart debug add FlavorName --- routers/repo/modelarts.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 0dc299a3b2..fe65386cb2 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -528,6 +528,8 @@ func NotebookManage(ctx *context.Context) { Description: task.Description, CreatedUnix: createTime, UpdatedUnix: createTime, + FlavorName: task.FlavorName, + FlavorCode: task.FlavorCode, } err = models.RestartCloudbrain(task, newTask) -- 2.34.1 From 6272f23799121c4725ab94a1e402aba0d1ad8c07 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 27 Jul 2022 09:18:03 +0800 Subject: [PATCH 2/4] fix-2357 --- models/cloudbrain.go | 32 ++++++++++++++++++++++++++++++++ models/cloudbrain_static.go | 15 --------------- routers/repo/cloudbrain.go | 28 ++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 23 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index c1f798ea83..c0ef455747 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1680,6 +1680,38 @@ func GetCloudbrainsNeededStopByUserID(userID int64) ([]*Cloudbrain, error) { return cloudBrains, err } +func GetWaittingTop() ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain.status": string(JobWaiting)}, + ) + sess.OrderBy("cloudbrain.created_unix ASC limit 1") + cloudbrains := make([]*CloudbrainInfo, 0, 1) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} +func GetModelartsReDebugTaskByJobId(jobID string) ([]*Cloudbrain, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain.job_id": jobID}, + ) + sess.OrderBy("cloudbrain.created_unix ASC limit 10") + // sess.OrderBy("cloudbrain.created_unix DSC limit 10") + cloudbrains := make([]*Cloudbrain, 0, 10) + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + func GetCloudbrainsNeededStopByRepoID(repoID int64) ([]*Cloudbrain, error) { cloudBrains := make([]*Cloudbrain, 0) err := x.Cols("job_id", "status", "type", "job_type", "version_id", "start_time").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 86143d9955..e3ac5e9632 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -211,21 +211,6 @@ func GetAllStatusCloudBrain() map[string]int { return cloudBrainStatusResult } -func GetWaittingTop() ([]*CloudbrainInfo, error) { - sess := x.NewSession() - defer sess.Close() - var cond = builder.NewCond() - cond = cond.And( - builder.Eq{"cloudbrain.status": string(JobWaiting)}, - ) - sess.OrderBy("cloudbrain.created_unix ASC limit 10") - cloudbrains := make([]*CloudbrainInfo, 0, 10) - if err := sess.Table(&Cloudbrain{}).Where(cond). - Find(&cloudbrains); err != nil { - log.Info("find error.") - } - return cloudbrains, nil -} func GetRunningTop() ([]*CloudbrainInfo, error) { sess := x.NewSession() defer sess.Close() diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d5a5e1a8ff..73c8582f14 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -3076,18 +3076,18 @@ func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) { ReplaceFlavorName := strings.ReplaceAll(task.FlavorName, "：", ":") return ReplaceFlavorName, nil } else if task.Type == models.TypeCloudBrainTwo && task.FlavorName == "" && task.FlavorCode != "" { - index := strings.LastIndex(task.FlavorCode, ".") - cardNum, err := strconv.Atoi(strings.TrimSpace(task.FlavorCode[index+1 : len(task.FlavorCode)])) + CloudbrainTwoFlavorName := getFlavorNameByFlavorCode(task.FlavorCode) + return CloudbrainTwoFlavorName, nil + } else if task.Type == models.TypeCloudBrainTwo && task.JobType == string(models.JobTypeDebug) && task.FlavorName == "" && task.FlavorCode == "" { + tasks, err := models.GetModelartsReDebugTaskByJobId(task.JobID) if err != nil { - log.Error("strconv.Atoi failed: %v", err) return "", err } - CloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(task.FlavorCode[index+1:len(task.FlavorCode)]) + - "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + - "核" + strconv.Itoa(cardNum*256) + "GB" - return CloudbrainTwoFlavorName, nil + if len(tasks) >= 1 { + return getFlavorNameByFlavorCode(tasks[0].FlavorCode), nil + } + return "", nil } - return "", nil } @@ -3112,3 +3112,15 @@ func getCloudBrainOneResourceSpec(task models.Cloudbrain) (*models.ResourceSpec, } return nil, nil, nil } +func getFlavorNameByFlavorCode(flavorCode string) string { + index := strings.LastIndex(flavorCode, ".") + cardNum, err := strconv.Atoi(strings.TrimSpace(flavorCode[index+1 : len(flavorCode)])) + if err != nil { + log.Error("strconv.Atoi failed: %v", err) + return "" + } + CloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(flavorCode[index+1:len(flavorCode)]) + + "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + + "核" + strconv.Itoa(cardNum*256) + "GB" + return CloudbrainTwoFlavorName +} -- 2.34.1 From 4835745f20bc756057a86b569ffbf5a1eae86964 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 27 Jul 2022 09:31:04 +0800 Subject: [PATCH 3/4] fix --- models/cloudbrain.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index c0ef455747..f74fdd8255 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1702,8 +1702,7 @@ func GetModelartsReDebugTaskByJobId(jobID string) ([]*Cloudbrain, error) { cond = cond.And( builder.Eq{"cloudbrain.job_id": jobID}, ) - sess.OrderBy("cloudbrain.created_unix ASC limit 10") - // sess.OrderBy("cloudbrain.created_unix DSC limit 10") + sess.OrderBy("cloudbrain.created_unix ASC limit 1") cloudbrains := make([]*Cloudbrain, 0, 10) if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { -- 2.34.1 From 40f320f5a153b809d56049b8515cd92492d6e3f4 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 27 Jul 2022 14:41:30 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E8=A7=84=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 60 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 73c8582f14..aa77e8d5c1 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2993,33 +2993,33 @@ func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { return "" } func GetCloudbrainCardDuration(task models.Cloudbrain) string { - CardNum, _, _ := GetCloudbrainCardNumAndType(task) - CardDuration := models.ConvertDurationToStr(int64(CardNum) * task.Duration) - return CardDuration + cardNum, _, _ := GetCloudbrainCardNumAndType(task) + cardDuration := models.ConvertDurationToStr(int64(cardNum) * task.Duration) + return cardDuration } func GetCloudbrainWaitTime(task models.Cloudbrain) string { - var WaitTime string + var waitTime string if task.Status == string(models.JobWaiting) { - WaitTimeInt := time.Now().Unix() - task.CreatedUnix.AsTime().Unix() - WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - WaitTime = "00:00:00" + waitTimeInt := time.Now().Unix() - task.CreatedUnix.AsTime().Unix() + waitTime = models.ConvertDurationToStr(waitTimeInt) + if waitTimeInt < 0 { + waitTime = "00:00:00" } } else if task.Status == string(models.JobStopped) && task.StartTime.AsTime().Unix() == 0 { - WaitTimeInt := task.EndTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() - WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - WaitTime = "00:00:00" + waitTimeInt := task.EndTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() + waitTime = models.ConvertDurationToStr(waitTimeInt) + if waitTimeInt < 0 { + waitTime = "00:00:00" } } else { - WaitTimeInt := task.StartTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() - WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - WaitTime = "00:00:00" + waitTimeInt := task.StartTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() + waitTime = models.ConvertDurationToStr(waitTimeInt) + if waitTimeInt < 0 { + waitTime = "00:00:00" } } - return WaitTime + return waitTime } func GetCloudbrainCardNumAndType(task models.Cloudbrain) (int, string, error) { @@ -3029,11 +3029,11 @@ func GetCloudbrainCardNumAndType(task models.Cloudbrain) (int, string, error) { if !models.GpuInfosMapInitFlag { models.InitCloudbrainOneGpuInfoMap() } - FlavorName, err := GetCloudbrainFlavorName(task) + flavorName, err := GetCloudbrainFlavorName(task) if err != nil { return 0, "", nil } - return getCardNumAndTypeByFlavorname(FlavorName) + return getCardNumAndTypeByFlavorname(flavorName) } func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { @@ -3058,26 +3058,26 @@ func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) { if task.Type == models.TypeCloudBrainOne { - ResourceSpec, GpuInfo, err := getCloudBrainOneResourceSpec(task) + resourceSpec, gpuInfo, err := getCloudBrainOneResourceSpec(task) if err != nil { log.Info("getCloudBrainOneResourceSpec err:", err) return "", err } else { - if ResourceSpec == nil || GpuInfo == nil { - err := errors.New("ResourceSpec or GpuInfo is nil") + if resourceSpec == nil || gpuInfo == nil { + err := errors.New("resourceSpec or gpuInfo is nil") return "", err } else { - CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(ResourceSpec.GpuNum) + "*Nvidia-" + GpuInfo.Value + - " | CPU:" + strconv.Itoa(ResourceSpec.CpuNum) + "核" + strconv.Itoa(ResourceSpec.MemMiB) + "MB" + CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(resourceSpec.GpuNum) + "*Nvidia-" + gpuInfo.Value + + " | CPU:" + strconv.Itoa(resourceSpec.CpuNum) + "核" + strconv.Itoa(resourceSpec.MemMiB) + "MB" return CloudbrainOneFlavorName, nil } } } else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeC2Net) && task.FlavorName != "" { - ReplaceFlavorName := strings.ReplaceAll(task.FlavorName, "：", ":") - return ReplaceFlavorName, nil + replaceFlavorName := strings.ReplaceAll(task.FlavorName, "：", ":") + return replaceFlavorName, nil } else if task.Type == models.TypeCloudBrainTwo && task.FlavorName == "" && task.FlavorCode != "" { - CloudbrainTwoFlavorName := getFlavorNameByFlavorCode(task.FlavorCode) - return CloudbrainTwoFlavorName, nil + cloudbrainTwoFlavorName := getFlavorNameByFlavorCode(task.FlavorCode) + return cloudbrainTwoFlavorName, nil } else if task.Type == models.TypeCloudBrainTwo && task.JobType == string(models.JobTypeDebug) && task.FlavorName == "" && task.FlavorCode == "" { tasks, err := models.GetModelartsReDebugTaskByJobId(task.JobID) if err != nil { @@ -3119,8 +3119,8 @@ func getFlavorNameByFlavorCode(flavorCode string) string { log.Error("strconv.Atoi failed: %v", err) return "" } - CloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(flavorCode[index+1:len(flavorCode)]) + + cloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(flavorCode[index+1:len(flavorCode)]) + "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + "核" + strconv.Itoa(cardNum*256) + "GB" - return CloudbrainTwoFlavorName + return cloudbrainTwoFlavorName } -- 2.34.1