|
- package repo
-
- import (
- "bufio"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "net/http"
- "os"
- "regexp"
- "sort"
- "strconv"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/timeutil"
- "github.com/unknwon/i18n"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/auth"
- "code.gitea.io/gitea/modules/base"
- "code.gitea.io/gitea/modules/cloudbrain"
- "code.gitea.io/gitea/modules/context"
- "code.gitea.io/gitea/modules/git"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/modelarts"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
- "code.gitea.io/gitea/modules/util"
- )
-
- const (
- tplCloudBrainNew base.TplName = "repo/cloudbrain/new"
- tplCloudBrainShow base.TplName = "repo/cloudbrain/show"
- tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index"
-
- tplCloudBrainBenchmarkIndex base.TplName = "repo/cloudbrain/benchmark/index"
- tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new"
- tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show"
-
- tplCloudBrainTrainJobNew base.TplName = "repo/cloudbrain/trainjob/new"
- tplCloudBrainTrainJobShow base.TplName = "repo/cloudbrain/trainjob/show"
- )
-
- var (
- gpuInfos *models.GpuInfos
- categories *models.Categories
- benchmarkTypes *models.BenchmarkTypes
- benchmarkGpuInfos *models.GpuInfos
- benchmarkResourceSpecs *models.ResourceSpecs
- trainGpuInfos *models.GpuInfos
- )
-
- const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types"
-
- var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0)
-
- var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
-
- // MustEnableDataset check if repository enable internal cb
- func MustEnableCloudbrain(ctx *context.Context) {
- if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
- ctx.NotFound("MustEnableCloudbrain", nil)
- return
- }
- }
-
- func cutString(str string, lens int) string {
- if len(str) < lens {
- return str
- }
- return str[:lens]
- }
-
- func jobNamePrefixValid(s string) string {
- lowStr := strings.ToLower(s)
- re := regexp.MustCompile(`[^a-z0-9_\\-]+`)
-
- removeSpecial := re.ReplaceAllString(lowStr, "")
-
- re = regexp.MustCompile(`^[_\\-]+`)
- return re.ReplaceAllString(removeSpecial, "")
-
- }
-
- func cloudBrainNewDataPrepare(ctx *context.Context) error {
- ctx.Data["PageIsCloudBrain"] = true
- t := time.Now()
- var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
- ctx.Data["display_job_name"] = displayJobName
-
- result, err := cloudbrain.GetImages()
- if err != nil {
- ctx.Data["error"] = err.Error()
- log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["MsgID"])
- }
-
- for i, payload := range result.Payload.ImageInfo {
- if strings.HasPrefix(result.Payload.ImageInfo[i].Place, "192.168") {
- result.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
- } else {
- result.Payload.ImageInfo[i].PlaceView = payload.Place
- }
- }
-
- ctx.Data["images"] = result.Payload.ImageInfo
-
- resultPublic, err := cloudbrain.GetPublicImages()
- if err != nil {
- ctx.Data["error"] = err.Error()
- log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["MsgID"])
- }
-
- for i, payload := range resultPublic.Payload.ImageInfo {
- if strings.HasPrefix(resultPublic.Payload.ImageInfo[i].Place, "192.168") {
- resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
- } else {
- resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place
- }
- }
-
- ctx.Data["public_images"] = resultPublic.Payload.ImageInfo
-
- attachs, err := models.GetAllUserAttachments(ctx.User.ID)
- if err != nil {
- log.Error("GetAllUserAttachments failed: %v", err, ctx.Data["MsgID"])
- return err
- }
-
- ctx.Data["attachments"] = attachs
- ctx.Data["command"] = cloudbrain.Command
- ctx.Data["code_path"] = cloudbrain.CodeMountPath
- ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
- ctx.Data["model_path"] = cloudbrain.ModelMountPath
- ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
- ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled
-
- if categories == nil {
- json.Unmarshal([]byte(setting.BenchmarkCategory), &categories)
- }
- ctx.Data["benchmark_categories"] = categories.Category
-
- ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType
-
- if gpuInfos == nil {
- json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
- }
- ctx.Data["gpu_types"] = gpuInfos.GpuInfo
-
- if trainGpuInfos == nil {
- json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos)
- }
- ctx.Data["train_gpu_types"] = trainGpuInfos.GpuInfo
-
- if benchmarkGpuInfos == nil {
- json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos)
- }
- ctx.Data["benchmark_gpu_types"] = benchmarkGpuInfos.GpuInfo
-
- if benchmarkResourceSpecs == nil {
- json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs)
- }
- ctx.Data["benchmark_resource_specs"] = benchmarkResourceSpecs.ResourceSpec
-
- if cloudbrain.ResourceSpecs == nil {
- json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
- }
- ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec
-
- if cloudbrain.TrainResourceSpecs == nil {
- json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs)
- }
- ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec
- ctx.Data["params"] = ""
- ctx.Data["branchName"] = ctx.Repo.BranchName
-
- ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath
- ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled
-
- ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath
- ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled
-
- ctx.Data["cloudbraintype"] = models.TypeCloudBrainOne
-
- return nil
- }
-
- func CloudBrainNew(ctx *context.Context) {
- err := cloudBrainNewDataPrepare(ctx)
- if err != nil {
- ctx.ServerError("get new cloudbrain info failed", err)
- return
- }
- ctx.HTML(200, tplCloudBrainNew)
- }
-
- func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
- ctx.Data["PageIsCloudBrain"] = true
- displayJobName := form.DisplayJobName
- jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
- image := form.Image
- uuid := form.Attachment
- jobType := form.JobType
- gpuQueue := form.GpuType
- codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
- resourceSpecId := form.ResourceSpecId
- branchName := form.BranchName
- repo := ctx.Repo.Repository
-
- tpl := tplCloudBrainNew
- command := cloudbrain.Command
- if jobType == string(models.JobTypeTrain) {
- tpl = tplCloudBrainTrainJobNew
- commandTrain, err := getTrainJobCommand(form)
- if err != nil {
- log.Error("getTrainJobCommand failed: %v", err)
- ctx.RenderWithErr(err.Error(), tpl, &form)
- return
- }
-
- command = commandTrain
- }
-
- tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
- if err == nil {
- if len(tasks) != 0 {
- log.Error("the job name did already exist", ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("the job name did already exist", tpl, &form)
- return
- }
- } else {
- if !models.IsErrJobNotExist(err) {
- log.Error("system error, %v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("system error", tpl, &form)
- return
- }
- }
-
- if !jobNamePattern.MatchString(displayJobName) {
- ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
- return
- }
-
- if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) && jobType != string(models.JobTypeTrain) {
- log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("jobtype error", tpl, &form)
- return
- }
-
- count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType)
- if err != nil {
- log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("system error", tpl, &form)
- return
- } else {
- if count >= 1 {
- log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("you have already a running or waiting task, can not create more", tpl, &form)
- return
- }
- }
-
- if branchName == "" {
- branchName = cloudbrain.DefaultBranchName
- }
- downloadCode(repo, codePath, branchName)
- uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
-
- modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/"
- mkModelPath(modelPath)
- uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")
-
- benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
- if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) {
- var gpuType string
- for _, gpuInfo := range gpuInfos.GpuInfo {
- if gpuInfo.Queue == gpuQueue {
- gpuType = gpuInfo.Value
- }
- }
- downloadRateCode(repo, jobName, setting.BenchmarkOwner, setting.BenchmarkName, benchmarkPath, form.BenchmarkCategory, gpuType)
- uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/")
- }
-
- snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
- if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
- downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "")
- uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/")
- }
-
- brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
- if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
- downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "")
- uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/")
- }
-
- err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
- storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
- storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
- storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, branchName, form.BootFile, form.Params,
- 0, 0, resourceSpecId)
- if err != nil {
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr(err.Error(), tpl, &form)
- return
- }
-
- if jobType == string(models.JobTypeTrain) {
- ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=all")
- } else {
- ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all")
- }
- }
-
- func CloudBrainRestart(ctx *context.Context) {
- var ID = ctx.Params(":id")
- var resultCode = "0"
- var errorMsg = ""
- var status = string(models.JobWaiting)
- task := ctx.Cloudbrain
- for {
- if task.Status != string(models.JobStopped) && task.Status != string(models.JobSucceeded) && task.Status != string(models.JobFailed) {
- log.Error("the job(%s) is not stopped", task.JobName, ctx.Data["MsgID"])
- resultCode = "-1"
- errorMsg = "the job is not stopped"
- break
- }
-
- if task.Image == "" || task.GpuQueue == "" || task.Type != models.TypeCloudBrainOne {
- log.Error("the job(%s) version is too old", task.JobName, ctx.Data["MsgID"])
- resultCode = "-1"
- errorMsg = "the job's version is too old and can not be restarted"
- break
- }
-
- if !ctx.IsSigned || (ctx.User.ID != task.UserID && !ctx.IsUserSiteAdmin()) {
- log.Error("the user has no right ro restart the job", task.JobName, ctx.Data["MsgID"])
- resultCode = "-1"
- errorMsg = "you have no right to restart the job"
- break
- }
-
- var hasSameResource bool
- if gpuInfos == nil {
- json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
- }
- for _, resourceType := range gpuInfos.GpuInfo {
- if resourceType.Queue == task.GpuQueue {
- hasSameResource = true
- continue
- }
- }
-
- if !hasSameResource {
- log.Error("has no same resource, can not restart", ctx.Data["MsgID"])
- resultCode = "-1"
- errorMsg = "the job's version is too old and can not be restarted"
- break
- }
-
- count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeDebug))
- if err != nil {
- log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
- resultCode = "-1"
- errorMsg = "system error"
- break
- } else {
- if count >= 1 {
- log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
- resultCode = "-1"
- errorMsg = "you have already a running or waiting task, can not create more"
- break
- }
- }
-
- err = cloudbrain.RestartTask(ctx, task, &ID)
- if err != nil {
- log.Error("RestartTask failed:%v", err.Error(), ctx.Data["MsgID"])
- resultCode = "-1"
- errorMsg = "system error"
- break
- }
-
- break
- }
-
- ctx.JSON(200, map[string]string{
- "result_code": resultCode,
- "error_msg": errorMsg,
- "status": status,
- "id": ID,
- })
- }
-
- func CloudBrainBenchMarkShow(ctx *context.Context) {
- cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark)
- }
-
- func CloudBrainShow(ctx *context.Context) {
- cloudBrainShow(ctx, tplCloudBrainShow, models.JobTypeDebug)
- }
-
- func CloudBrainTrainJobShow(ctx *context.Context) {
- cloudBrainShow(ctx, tplCloudBrainTrainJobShow, models.JobTypeTrain)
- }
-
- func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.JobType) {
- ctx.Data["PageIsCloudBrain"] = true
- debugListType := ctx.Query("debugListType")
-
- var task *models.Cloudbrain
- var err error
- if jobType == models.JobTypeTrain {
- task, err = models.GetCloudbrainByJobID(ctx.Params(":jobid"))
- } else {
- task, err = models.GetCloudbrainByIDWithDeleted(ctx.Params(":id"))
- }
-
- if err != nil {
- log.Info("error:" + err.Error())
- ctx.Data["error"] = err.Error()
- return
- }
-
- result, err := cloudbrain.GetJob(task.JobID)
- if err != nil {
- log.Info("error:" + err.Error())
- ctx.Data["error"] = err.Error()
- return
- }
-
- if task.JobType == string(models.JobTypeTrain) {
- if cloudbrain.TrainResourceSpecs == nil {
- json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs)
- }
- for _, tmp := range cloudbrain.TrainResourceSpecs.ResourceSpec {
- if tmp.Id == task.ResourceSpecId {
- ctx.Data["GpuNum"] = tmp.GpuNum
- ctx.Data["CpuNum"] = tmp.CpuNum
- ctx.Data["MemMiB"] = tmp.MemMiB
- ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
- }
- }
- } else {
- if cloudbrain.ResourceSpecs == nil {
- json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
- }
- for _, tmp := range cloudbrain.ResourceSpecs.ResourceSpec {
- if tmp.Id == task.ResourceSpecId {
- ctx.Data["GpuNum"] = tmp.GpuNum
- ctx.Data["CpuNum"] = tmp.CpuNum
- ctx.Data["MemMiB"] = tmp.MemMiB
- ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
- }
- }
- }
-
- if result != nil {
- jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
- jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB")
- spec := "GPU数:" + strconv.Itoa(jobRes.Resource.NvidiaComGpu) + ",CPU数:" + strconv.Itoa(jobRes.Resource.CPU) + ",内存(MB):" + jobRes.Resource.Memory
- ctx.Data["resource_spec"] = spec
- if task.JobType == string(models.JobTypeTrain) {
- if trainGpuInfos == nil {
- json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos)
- }
- for _, resourceType := range trainGpuInfos.GpuInfo {
- if resourceType.Queue == jobRes.Config.GpuType {
- ctx.Data["resource_type"] = resourceType.Value
- }
- }
- } else {
- if gpuInfos == nil {
- json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
- }
- for _, resourceType := range gpuInfos.GpuInfo {
- if resourceType.Queue == jobRes.Config.GpuType {
- ctx.Data["resource_type"] = resourceType.Value
- }
- }
- }
- taskRoles := jobRes.TaskRoles
- if jobRes.JobStatus.State != string(models.JobFailed) {
-
- taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
- ctx.Data["taskRes"] = taskRes
- task.Status = taskRes.TaskStatuses[0].State
- task.ContainerID = taskRes.TaskStatuses[0].ContainerID
- task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
- models.ParseAndSetDurationFromCloudBrainOne(jobRes, task)
-
- if task.DeletedAt.IsZero() { //normal record
- err = models.UpdateJob(task)
- if err != nil {
- ctx.Data["error"] = err.Error()
- return
- }
- } else { //deleted record
-
- }
- } else {
- task.Status = jobRes.JobStatus.State
- taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{
- {
- State: jobRes.JobStatus.State,
- },
- }}
- ctx.Data["taskRes"] = taskRes
- jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05")
- jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05")
- }
-
- ctx.Data["result"] = jobRes
- } else {
- log.Info("error:" + err.Error())
- return
- }
-
- user, err := models.GetUserByID(task.UserID)
- if err == nil {
- task.User = user
- }
-
- if task.BenchmarkTypeID > 0 {
- for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
- if task.BenchmarkTypeID == benchmarkType.Id {
- ctx.Data["BenchmarkTypeName"] = benchmarkType.First
- for _, benchmarkChildType := range benchmarkType.Second {
- if task.BenchmarkChildTypeID == benchmarkChildType.Id {
- ctx.Data["BenchmarkChildTypeName"] = benchmarkChildType.Value
- break
- }
- }
- break
- }
- }
- }
- if task.TrainJobDuration == "" {
- if task.Duration == 0 {
- var duration int64
- if task.Status == string(models.JobRunning) {
- duration = time.Now().Unix() - int64(task.CreatedUnix)
- } else {
- duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix)
- }
- task.Duration = duration
- }
- task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
- }
- ctx.Data["duration"] = task.TrainJobDuration
-
- if len(task.Parameters) > 0 {
- var parameters models.Parameters
-
- err := json.Unmarshal([]byte(task.Parameters), ¶meters)
- if err != nil {
- log.Error("Failed to Unmarshal Parameters: %s (%v)", task.Parameters, err)
- task.Parameters = ""
- } else {
- if len(parameters.Parameter) > 0 {
- paramTemp := ""
- for _, Parameter := range parameters.Parameter {
- param := Parameter.Label + " = " + Parameter.Value + "; "
- paramTemp = paramTemp + param
- }
- task.Parameters = paramTemp[:len(paramTemp)-2]
- } else {
- task.Parameters = ""
- }
- }
-
- }
- attachment, err := models.GetAttachmentByUUID(task.Uuid)
- if err == nil {
- ctx.Data["datasetname"] = attachment.Name
- } else {
- ctx.Data["datasetname"] = ""
- }
-
- ctx.Data["task"] = task
- ctx.Data["jobName"] = task.JobName
- ctx.Data["displayJobName"] = task.DisplayJobName
- version_list_task := make([]*models.Cloudbrain, 0)
- version_list_task = append(version_list_task, task)
- ctx.Data["version_list_task"] = version_list_task
- ctx.Data["debugListType"] = debugListType
- ctx.Data["code_path"] = cloudbrain.CodeMountPath
- ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
- ctx.Data["model_path"] = cloudbrain.ModelMountPath
- ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task)
- ctx.HTML(200, tpName)
- }
-
- func CloudBrainDebug(ctx *context.Context) {
- task := ctx.Cloudbrain
- debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
- ctx.Redirect(debugUrl)
- }
-
- func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
- err := cloudbrain.CommitImage(ctx.Cloudbrain.JobID, models.CommitImageParams{
- Ip: ctx.Cloudbrain.ContainerIp,
- TaskContainerId: ctx.Cloudbrain.ContainerID,
- ImageDescription: form.Description,
- ImageTag: form.Tag,
- })
- if err != nil {
- log.Error("CommitImage(%s) failed:%v", ctx.Cloudbrain.JobName, err.Error(), ctx.Data["msgID"])
- ctx.JSON(200, map[string]string{
- "result_code": "-1",
- "error_msg": "CommitImage failed",
- })
- return
- }
-
- ctx.JSON(200, map[string]string{
- "result_code": "0",
- "error_msg": "",
- })
- }
-
- func CloudBrainStop(ctx *context.Context) {
- var ID = ctx.Params(":id")
- var resultCode = "0"
- var errorMsg = ""
- var status = ""
-
- task := ctx.Cloudbrain
- for {
- if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) || task.Status == string(models.JobSucceeded) {
- log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
- resultCode = "-1"
- errorMsg = "system error"
- break
- }
-
- err := cloudbrain.StopJob(task.JobID)
- if err != nil {
- log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
- resultCode = "-1"
- errorMsg = "system error"
- break
- }
-
- task.Status = string(models.JobStopped)
- if task.EndTime == 0 {
- task.EndTime = timeutil.TimeStampNow()
- }
- task.ComputeAndSetDuration()
- err = models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
- resultCode = "-1"
- errorMsg = "system error"
- break
- }
-
- status = task.Status
- break
- }
-
- ctx.JSON(200, map[string]interface{}{
- "result_code": resultCode,
- "error_msg": errorMsg,
- "status": status,
- "id": ID,
- "StatusOK": 0,
- })
- }
-
- func StopJobsByUserID(userID int64) {
- cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID)
- if err != nil {
- log.Warn("Failed to get cloudBrain info", err)
- return
- }
- StopJobs(cloudBrains)
-
- }
-
- func StopJobsByRepoID(repoID int64) {
- cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID)
- if err != nil {
- log.Warn("Failed to get cloudBrain info", err)
- return
- }
- StopJobs(cloudBrains)
- }
-
- /**
-
- */
- func StopJobs(cloudBrains []*models.Cloudbrain) {
-
- for _, taskInfo := range cloudBrains {
-
- if taskInfo.Type == models.TypeCloudBrainOne {
- err := retry(3, time.Second*30, func() error {
- return cloudbrain.StopJob(taskInfo.JobID)
- })
-
- logErrorAndUpdateJobStatus(err, taskInfo)
- } else {
- if taskInfo.JobType == string(models.JobTypeTrain) {
- err := retry(3, time.Second*30, func() error {
- _, err := modelarts.StopTrainJob(taskInfo.JobID, strconv.FormatInt(taskInfo.VersionID, 10))
- return err
- })
- logErrorAndUpdateJobStatus(err, taskInfo)
- } else {
- param := models.NotebookAction{
- Action: models.ActionStop,
- }
- err := retry(3, time.Second*30, func() error {
- _, err := modelarts.ManageNotebook(taskInfo.JobID, param)
- return err
- })
- logErrorAndUpdateJobStatus(err, taskInfo)
- }
- }
-
- }
- }
-
- func retry(attempts int, sleep time.Duration, f func() error) (err error) {
- for i := 0; i < attempts; i++ {
- if i > 0 {
- log.Warn("retrying after error:", err)
- time.Sleep(sleep)
- }
- err = f()
- if err == nil {
- return nil
- }
- }
- return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
- }
-
- func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
- if err != nil {
- log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
- } else {
- taskInfo.Status = string(models.JobStopped)
- if taskInfo.EndTime == 0 {
- taskInfo.EndTime = timeutil.TimeStampNow()
- }
- taskInfo.ComputeAndSetDuration()
- err = models.UpdateJob(taskInfo)
- if err != nil {
- log.Warn("UpdateJob failed", err)
- }
- }
- }
-
- func CloudBrainDel(ctx *context.Context) {
- var listType = ctx.Query("debugListType")
- if err := deleteCloudbrainJob(ctx); err != nil {
- log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
- ctx.ServerError(err.Error(), err)
- return
- }
-
- var isAdminPage = ctx.Query("isadminpage")
- if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
- ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
- } else {
- ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=" + listType)
- }
- }
-
- func deleteCloudbrainJob(ctx *context.Context) error {
- task := ctx.Cloudbrain
-
- if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) && task.Status != string(models.JobSucceeded) {
- log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
- return errors.New("the job has not been stopped")
- }
-
- err := models.DeleteJob(task)
- if err != nil {
- log.Error("DeleteJob failed: %v", err, ctx.Data["msgID"])
- return err
- }
-
- deleteJobStorage(task.JobName, models.TypeCloudBrainOne)
-
- return nil
- }
-
- func CloudBrainShowModels(ctx *context.Context) {
- ctx.Data["PageIsCloudBrain"] = true
-
- ID := ctx.Params(":id")
- parentDir := ctx.Query("parentDir")
- dirArray := strings.Split(parentDir, "/")
- task, err := models.GetCloudbrainByID(ID)
- if err != nil {
- log.Error("no such job!", ctx.Data["msgID"])
- ctx.ServerError("no such job:", err)
- return
- }
-
- //get dirs
- dirs, err := GetModelDirs(task.JobName, parentDir)
- if err != nil {
- log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
- ctx.ServerError("GetModelDirs failed:", err)
- return
- }
-
- var fileInfos []storage.FileInfo
- err = json.Unmarshal([]byte(dirs), &fileInfos)
- if err != nil {
- log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
- ctx.ServerError("json.Unmarshal failed:", err)
- return
- }
-
- for i, fileInfo := range fileInfos {
- temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
- fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
- }
-
- sort.Slice(fileInfos, func(i, j int) bool {
- return fileInfos[i].ModTime > fileInfos[j].ModTime
- })
-
- ctx.Data["Path"] = dirArray
- ctx.Data["Dirs"] = fileInfos
- ctx.Data["task"] = task
- ctx.Data["ID"] = ID
- ctx.HTML(200, tplCloudBrainShowModels)
- }
-
- func GetPublicImages(ctx *context.Context) {
-
- getImages(ctx, cloudbrain.Public)
-
- }
-
- func GetCustomImages(ctx *context.Context) {
-
- getImages(ctx, cloudbrain.Custom)
-
- }
-
- func getImages(ctx *context.Context, imageType string) {
- log.Info("Get images begin")
-
- page := ctx.QueryInt("page")
- size := ctx.QueryInt("size")
- name := ctx.Query("name")
- getImagesResult, err := cloudbrain.GetImagesPageable(page, size, imageType, name)
- if err != nil {
- log.Error("Can not get images:%v", err)
- ctx.JSON(http.StatusOK, models.GetImagesPayload{
- Count: 0,
- TotalPages: 0,
- ImageInfo: []*models.ImageInfo{},
- })
- } else {
- ctx.JSON(http.StatusOK, getImagesResult.Payload)
- }
- log.Info("Get images end")
- }
-
- func GetModelDirs(jobName string, parentDir string) (string, error) {
- var req string
- modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/")
- if parentDir == "" {
- req = "baseDir=" + modelActualPath
- } else {
- req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
- }
-
- return getDirs(req)
- }
-
- func CloudBrainDownloadModel(ctx *context.Context) {
- parentDir := ctx.Query("parentDir")
- fileName := ctx.Query("fileName")
- jobName := ctx.Query("jobName")
- filePath := "jobs/" + jobName + "/model/" + parentDir
- url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
- if err != nil {
- log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
- ctx.ServerError("PresignedGetURL", err)
- return
- }
- http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
- }
-
- func GetRate(ctx *context.Context) {
- isObjectDetcionAll := ctx.QueryBool("isObjectDetcionAll")
- if isObjectDetcionAll {
- ctx.Redirect(setting.BenchmarkServerHost + "?username=admin")
- return
- }
-
- var ID = ctx.Params(":id")
- job, err := models.GetCloudbrainByID(ID)
- if err != nil {
- ctx.ServerError("GetCloudbrainByJobID failed", err)
- return
- }
-
- if job.JobType == string(models.JobTypeBenchmark) {
- log.Info("url=" + setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
- ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
- } else if job.JobType == string(models.JobTypeSnn4imagenet) {
- ctx.Redirect(setting.Snn4imagenetServerHost)
- } else if job.JobType == string(models.JobTypeBrainScore) {
- ctx.Redirect(setting.BrainScoreServerHost)
- } else {
- log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
- }
- }
-
- func downloadCode(repo *models.Repository, codePath, branchName string) error {
- if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName}); err != nil {
- log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
- return err
- }
-
- configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
- if err != nil {
- log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
- return err
- }
-
- defer configFile.Close()
-
- pos := int64(0)
- reader := bufio.NewReader(configFile)
- for {
- line, err := reader.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- log.Error("not find the remote-url")
- return nil
- } else {
- log.Error("read error: %v", err)
- return err
- }
- }
-
- if strings.Contains(line, "url") && strings.Contains(line, ".git") {
- originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n"
- if len(line) > len(originUrl) {
- originUrl += strings.Repeat(" ", len(line)-len(originUrl))
- }
- bytes := []byte(originUrl)
- _, err := configFile.WriteAt(bytes, pos)
- if err != nil {
- log.Error("WriteAt failed:%v", err)
- return err
- }
- break
- }
-
- pos += int64(len(line))
- }
-
- return nil
- }
-
- func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType string) error {
- err := os.MkdirAll(codePath, os.ModePerm)
- if err != nil {
- log.Error("mkdir codePath failed", err.Error())
- return err
- }
-
- repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName)
- if err != nil {
- log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error())
- return err
- }
-
- if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil {
- log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err)
- return err
- }
-
- fileName := codePath + cloudbrain.TaskInfoName
- f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
- if err != nil {
- log.Error("OpenFile failed", err.Error())
- return err
- }
-
- defer f.Close()
-
- data, err := json.Marshal(models.TaskInfo{
- Username: repo.Owner.Name,
- TaskName: taskName,
- CodeName: repo.Name,
- BenchmarkCategory: strings.Split(benchmarkCategory, ","),
- CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"),
- GpuType: gpuType,
- })
- if err != nil {
- log.Error("json.Marshal failed", err.Error())
- return err
- }
-
- _, err = f.Write(data)
- if err != nil {
- log.Error("WriteString failed", err.Error())
- return err
- }
-
- return nil
- }
-
- func uploadCodeToMinio(codePath, jobName, parentDir string) error {
- files, err := readDir(codePath)
- if err != nil {
- log.Error("readDir(%s) failed: %s", codePath, err.Error())
- return err
- }
-
- for _, file := range files {
- if file.IsDir() {
- if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil {
- log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error())
- return err
- }
- } else {
- destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name()
- sourceFile := codePath + file.Name()
- err = storage.Attachments.UploadObject(destObject, sourceFile)
- if err != nil {
- log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error())
- return err
- }
- }
- }
-
- return nil
- }
-
- func mkModelPath(modelPath string) error {
- err := os.MkdirAll(modelPath, os.ModePerm)
- if err != nil {
- log.Error("MkdirAll(%s) failed:%v", modelPath, err)
- return err
- }
-
- fileName := modelPath + "README"
- f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
- if err != nil {
- log.Error("OpenFile failed", err.Error())
- return err
- }
-
- defer f.Close()
-
- _, err = f.WriteString("You can put the model file into this directory and download it by the web page.")
- if err != nil {
- log.Error("WriteString failed", err.Error())
- return err
- }
-
- return nil
- }
-
- func deleteJobStorage(jobName string, cloudbrainType int) error {
- //delete local
- localJobPath := setting.JobPath + jobName
- err := os.RemoveAll(localJobPath)
- if err != nil {
- log.Error("RemoveAll(%s) failed:%v", localJobPath, err)
- }
-
- //delete oss
- if cloudbrainType == models.TypeCloudBrainOne {
- dirPath := setting.CBCodePathPrefix + jobName + "/"
- err = storage.Attachments.DeleteDir(dirPath)
- if err != nil {
- log.Error("DeleteDir(%s) failed:%v", localJobPath, err)
- }
- } else if cloudbrainType == models.TypeCloudBrainTwo {
- //dirPath := setting.CodePathPrefix + jobName + "/"
- //err = storage.ObsRemoveObject(setting.Bucket, dirPath)
- //if err != nil {
- // log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err)
- //}
- log.Info("no need to delete")
- } else {
- log.Error("cloudbrainType(%d) error", cloudbrainType)
- }
-
- return nil
- }
-
- func SyncCloudbrainStatus() {
- cloudBrains, err := models.GetCloudBrainUnStoppedJob()
- if err != nil {
- log.Error("GetCloudBrainUnStoppedJob failed:", err.Error())
- return
- }
-
- for _, task := range cloudBrains {
- if task.Type == models.TypeCloudBrainOne {
- result, err := cloudbrain.GetJob(task.JobID)
- if err != nil {
- log.Error("GetJob(%s) failed:%v", task.JobName, err)
- continue
- }
-
- if result != nil {
- jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
- taskRoles := jobRes.TaskRoles
- taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
- task.Status = taskRes.TaskStatuses[0].State
- if task.Status != string(models.JobWaiting) {
- models.ParseAndSetDurationFromCloudBrainOne(jobRes, task)
- err = models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
- }
-
- var maxDuration int64
- if task.JobType == string(models.JobTypeBenchmark) {
- maxDuration = setting.BenchmarkMaxDuration
- } else {
- maxDuration = setting.MaxDuration
- }
-
- if task.Duration >= maxDuration {
- log.Info("begin to stop job(%s), because of the duration", task.JobName)
- err = cloudbrain.StopJob(task.JobID)
- if err != nil {
- log.Error("StopJob(%s) failed:%v", task.JobName, err)
- continue
- }
- task.Status = string(models.JobStopped)
- if task.EndTime == 0 {
- task.EndTime = timeutil.TimeStampNow()
- }
- task.ComputeAndSetDuration()
- err = models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
- }
- }
- }
- }
- } else if task.Type == models.TypeCloudBrainTwo {
- if task.JobType == string(models.JobTypeDebug) {
- //result, err := modelarts.GetJob(task.JobID)
- result, err := modelarts.GetNotebook2(task.JobID)
- if err != nil {
- log.Error("GetJob(%s) failed:%v", task.JobName, err)
- continue
- }
-
- if result != nil {
- task.Status = result.Status
- if task.StartTime == 0 && result.Lease.UpdateTime > 0 {
- task.StartTime = timeutil.TimeStamp(result.Lease.UpdateTime / 1000)
- }
- if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) {
- task.EndTime = timeutil.TimeStampNow()
- }
- task.CorrectCreateUnix()
- task.ComputeAndSetDuration()
- err = models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
- continue
- }
- }
- } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) {
- result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
- if err != nil {
- log.Error("GetTrainJob(%s) failed:%v", task.JobName, err)
- continue
- }
-
- if result != nil {
- task.Status = modelarts.TransTrainJobStatus(result.IntStatus)
- task.Duration = result.Duration / 1000
- task.TrainJobDuration = result.TrainJobDuration
-
- if task.StartTime == 0 && result.StartTime > 0 {
- task.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
- }
- task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
- if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
- task.EndTime = task.StartTime.Add(task.Duration)
- }
- task.CorrectCreateUnix()
- err = models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
- continue
- }
- }
- } else {
- log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType)
- }
-
- } else {
- log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
- }
- }
-
- return
- }
-
- func HandleTaskWithNoDuration(ctx *context.Context) {
- log.Info("HandleTaskWithNoDuration start")
- count := 0
- start := time.Now().Unix()
- for {
- cloudBrains, err := models.GetStoppedJobWithNoDurationJob()
- if err != nil {
- log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error())
- break
- }
- if len(cloudBrains) == 0 {
- log.Info("HandleTaskWithNoTrainJobDuration:no task need handle")
- break
- }
- handleNoDurationTask(cloudBrains)
- count += len(cloudBrains)
- if len(cloudBrains) < 100 {
- log.Info("HandleTaskWithNoTrainJobDuration:task less than 100")
- break
- }
- if time.Now().Unix()-start > 600 {
- log.Info("HandleTaskWithNoDuration : time out")
- ctx.JSON(200, fmt.Sprintf("task stop for time out,count=%d", count))
- return
- }
- }
- log.Info("HandleTaskWithNoTrainJobDuration:count=%d", count)
- ctx.JSON(200, fmt.Sprintf("success,count=%d", count))
- }
-
- func handleNoDurationTask(cloudBrains []*models.Cloudbrain) {
- for _, task := range cloudBrains {
- time.Sleep(time.Millisecond * 100)
- log.Info("Handle job ,%+v", task)
- if task.Type == models.TypeCloudBrainOne {
- result, err := cloudbrain.GetJob(task.JobID)
- if err != nil {
- log.Error("GetJob(%s) failed:%v", task.JobName, err)
- updateDefaultDuration(task)
- continue
- }
-
- if result != nil {
- if result.Msg != "success" {
- updateDefaultDuration(task)
- continue
- }
- jobRes, err := models.ConvertToJobResultPayload(result.Payload)
- if err != nil || len(jobRes.TaskRoles) == 0 {
- updateDefaultDuration(task)
- continue
- }
- taskRoles := jobRes.TaskRoles
- taskRes, err := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
- if err != nil || len(taskRes.TaskStatuses) == 0 {
- updateDefaultDuration(task)
- continue
- }
- task.Status = taskRes.TaskStatuses[0].State
- log.Info("task startTime = %v endTime= %v ,jobId=%d", jobRes.JobStatus.StartTime, jobRes.JobStatus.EndTime, task.ID)
- if jobRes.JobStatus.CreatedTime > 0 {
- task.StartTime = timeutil.TimeStamp(jobRes.JobStatus.CreatedTime / 1000)
- if jobRes.JobStatus.CompletedTime > 0 {
- task.EndTime = timeutil.TimeStamp(jobRes.JobStatus.CompletedTime / 1000)
- } else {
- task.EndTime = task.UpdatedUnix
- }
- } else {
- task.StartTime = 0
- task.EndTime = 0
- }
-
- if task.EndTime < task.StartTime {
- log.Info("endTime[%v] is less than starTime[%v],jobId=%d", task.EndTime, task.StartTime, task.ID)
- st := task.StartTime
- task.StartTime = task.EndTime
- task.EndTime = st
- }
- task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix()
- task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
- err = models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
- }
- }
- } else if task.Type == models.TypeCloudBrainTwo {
- if task.JobType == string(models.JobTypeDebug) {
- //result, err := modelarts.GetJob(task.JobID)
- result, err := modelarts.GetNotebook2(task.JobID)
- if err != nil {
- log.Error("GetJob(%s) failed:%v", task.JobName, err)
- updateDefaultDuration(task)
- continue
- }
-
- if result != nil {
- task.Status = result.Status
- startTime := result.Lease.CreateTime
- duration := result.Lease.Duration / 1000
- if startTime > 0 {
- task.StartTime = timeutil.TimeStamp(startTime / 1000)
- task.EndTime = task.StartTime.Add(duration)
- }
- task.CorrectCreateUnix()
- task.ComputeAndSetDuration()
- err = models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
- continue
- }
- }
- } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) {
- result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
- if err != nil {
- log.Error("GetTrainJob(%s) failed:%v", task.JobName, err)
- updateDefaultDuration(task)
- continue
- }
-
- if result != nil {
- startTime := result.StartTime / 1000
- if startTime > 0 {
- task.StartTime = timeutil.TimeStamp(startTime)
- task.EndTime = task.StartTime.Add(result.Duration / 1000)
- }
- task.ComputeAndSetDuration()
- err = models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
- continue
- }
- }
- } else {
- log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType)
- }
-
- } else {
- log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
- }
- }
- }
-
- func updateDefaultDuration(task *models.Cloudbrain) {
- log.Info("updateDefaultDuration: taskId=%d", task.ID)
- task.StartTime = task.CreatedUnix
- task.EndTime = task.UpdatedUnix
- task.ComputeAndSetDuration()
- err := models.UpdateJob(task)
- if err != nil {
- log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
- }
- }
-
- func CloudBrainBenchmarkIndex(ctx *context.Context) {
- MustEnableCloudbrain(ctx)
- repo := ctx.Repo.Repository
- page := ctx.QueryInt("page")
- if page <= 0 {
- page = 1
- }
-
- var jobTypes []string
- jobTypes = append(jobTypes, string(models.JobTypeBenchmark))
- ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
- ListOptions: models.ListOptions{
- Page: page,
- PageSize: setting.UI.IssuePagingNum,
- },
- RepoID: repo.ID,
- Type: models.TypeCloudBrainOne,
- JobTypes: jobTypes,
- })
- if err != nil {
- ctx.ServerError("Get debugjob faild:", err)
- return
- }
-
- for i, task := range ciTasks {
- ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
- ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource
- if ciTasks[i].TrainJobDuration == "" {
- if ciTasks[i].Duration == 0 {
- var duration int64
- if task.Status == string(models.JobRunning) {
- duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix)
- } else {
- duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix)
- }
- ciTasks[i].Duration = duration
- }
- ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(ciTasks[i].Duration)
- }
-
- ciTasks[i].BenchmarkTypeName = ""
- if task.BenchmarkTypeID > 0 {
- for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
- if task.BenchmarkTypeID == benchmarkType.Id {
- ciTasks[i].BenchmarkTypeRankLink = benchmarkType.RankLink
- ciTasks[i].BenchmarkTypeName = benchmarkType.First
- break
- }
- }
- }
- }
-
- pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
-
- ctx.Data["Page"] = pager
- ctx.Data["PageIsCloudBrain"] = true
- ctx.Data["Tasks"] = ciTasks
- ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx)
- ctx.Data["RepoIsEmpty"] = repo.IsEmpty
- ctx.HTML(200, tplCloudBrainBenchmarkIndex)
- }
-
- func GetChildTypes(ctx *context.Context) {
- benchmarkTypeID := ctx.QueryInt("benchmark_type_id")
- re := make(map[string]interface{})
- for {
- var isExist bool
- for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
- if benchmarkTypeID == benchmarkType.Id {
- isExist = true
- re["child_types"] = benchmarkType.Second
- re["result_code"] = "0"
- break
- }
- }
- if !isExist {
- re["result_code"] = "1"
- log.Error("no such benchmark_type_id", ctx.Data["MsgID"])
- re["errMsg"] = "system error"
- break
- }
- break
- }
- ctx.JSON(200, re)
- }
-
- func CloudBrainBenchmarkNew(ctx *context.Context) {
- ctx.Data["description"] = ""
- ctx.Data["benchmarkTypeID"] = -1
- ctx.Data["benchmark_child_types_id_hidden"] = -1
- err := cloudBrainNewDataPrepare(ctx)
- if err != nil {
- ctx.ServerError("get new cloudbrain info failed", err)
- return
- }
- ctx.HTML(200, tplCloudBrainBenchmarkNew)
- }
-
- func getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID int, ctx *context.Context) (*models.BenchmarkDataset, error) {
- var childInfo *models.BenchmarkDataset
-
- var isExist bool
- for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
- if benchmarkType.Id == benchmarkTypeID {
- for _, childType := range benchmarkType.Second {
- if childType.Id == benchmarkChildTypeID {
- childInfo = childType
- isExist = true
- break
- }
- }
- break
- }
- }
-
- if !isExist {
- log.Error("no such benchmark_type_id&benchmark_child_type_id")
- return childInfo, errors.New("no such benchmark_type_id&benchmark_child_type_id")
- }
-
- return childInfo, nil
- }
-
- func getBenchmarkGpuQueue(gpuQueue string) (string, error) {
- queue := ""
- if benchmarkGpuInfos == nil {
- if err := json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos); err != nil {
- log.Error("json.Unmarshal BenchmarkGpuTypes(%s) failed:%v", setting.BenchmarkGpuTypes, err)
- return queue, err
- }
- }
-
- var isExist bool
- for _, gpuInfo := range benchmarkGpuInfos.GpuInfo {
- if gpuQueue == gpuInfo.Queue {
- isExist = true
- queue = gpuQueue
- break
- }
- }
-
- if !isExist {
- log.Error("no such gpuQueue, %s", gpuQueue)
- return queue, errors.New("no such gpuQueue")
- }
-
- return queue, nil
- }
-
- func getBenchmarkResourceSpec(resourceSpecID int) (int, error) {
- var id int
- if benchmarkResourceSpecs == nil {
- if err := json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs); err != nil {
- log.Error("json.Unmarshal BenchmarkResourceSpecs(%s) failed:%v", setting.BenchmarkResourceSpecs, err)
- return id, err
- }
- }
-
- var isExist bool
- for _, resourceSpec := range benchmarkResourceSpecs.ResourceSpec {
- if resourceSpecID == resourceSpec.Id {
- isExist = true
- id = resourceSpecID
- break
- }
- }
-
- if !isExist {
- log.Error("no such resourceSpecID, %d", resourceSpecID)
- return id, errors.New("no such resourceSpec")
- }
-
- return id, nil
- }
-
- func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
- ctx.Data["PageIsCloudBrain"] = true
- displayJobName := form.DisplayJobName
- jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
- image := form.Image
- gpuQueue := form.GpuType
- command := cloudbrain.CommandBenchmark
- codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
- resourceSpecId := cloudbrain.BenchMarkResourceID
- benchmarkTypeID := form.BenchmarkTypeID
- benchmarkChildTypeID := form.BenchmarkChildTypeID
-
- ctx.Data["description"] = form.Description
- ctx.Data["benchmarkTypeID"] = benchmarkTypeID
- ctx.Data["benchmark_child_types_id_hidden"] = benchmarkChildTypeID
-
- repo := ctx.Repo.Repository
-
- tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName)
- if err == nil {
- if len(tasks) != 0 {
- log.Error("the job name did already exist", ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("the job name did already exist", tplCloudBrainBenchmarkNew, &form)
- return
- }
- } else {
- if !models.IsErrJobNotExist(err) {
- log.Error("system error, %v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
- return
- }
- }
-
- if !jobNamePattern.MatchString(jobName) {
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, &form)
- return
- }
-
- childInfo, err := getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID, ctx)
- if err != nil {
- log.Error("getBenchmarkAttachment failed:%v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("benchmark type error", tplCloudBrainBenchmarkNew, &form)
- return
- }
-
- _, err = getBenchmarkGpuQueue(gpuQueue)
- if err != nil {
- log.Error("getBenchmarkGpuQueue failed:%v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("gpu queue error", tplCloudBrainBenchmarkNew, &form)
- return
- }
-
- _, err = getBenchmarkResourceSpec(resourceSpecId)
- if err != nil {
- log.Error("getBenchmarkResourceSpec failed:%v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("resource spec error", tplCloudBrainBenchmarkNew, &form)
- return
- }
-
- count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeBenchmark))
- if err != nil {
- log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
- return
- } else {
- if count >= 1 {
- log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplCloudBrainBenchmarkNew, &form)
- return
- }
- }
-
- os.RemoveAll(codePath)
- if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil {
- log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
- return
- }
-
- if _, err := os.Stat(codePath + "/train.py"); err != nil {
- if os.IsNotExist(err) {
- // file does not exist
- log.Error("train.py does not exist, %v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("train.py does not exist", tplCloudBrainBenchmarkNew, &form)
- } else {
- log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
- }
- return
- } else if _, err := os.Stat(codePath + "/test.py"); err != nil {
- if os.IsNotExist(err) {
- // file does not exist
- log.Error("test.py does not exist, %v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("test.py does not exist", tplCloudBrainBenchmarkNew, &form)
- } else {
- log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
- }
- return
- }
-
- if err := uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil {
- log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
- return
- }
-
- benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
- var gpuType string
- for _, gpuInfo := range gpuInfos.GpuInfo {
- if gpuInfo.Queue == gpuQueue {
- gpuType = gpuInfo.Value
- }
- }
-
- if err := downloadRateCode(repo, jobName, childInfo.Owner, childInfo.RepoName, benchmarkPath, form.BenchmarkCategory, gpuType); err != nil {
- log.Error("downloadRateCode failed, %v", err, ctx.Data["MsgID"])
- //cloudBrainNewDataPrepare(ctx)
- //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
- //return
- }
-
- if err := uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/"); err != nil {
- log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
- //cloudBrainNewDataPrepare(ctx)
- //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
- //return
- }
-
- err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, childInfo.Attachment, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
- storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
- storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
- storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), string(models.JobTypeBenchmark), gpuQueue, form.Description, cloudbrain.DefaultBranchName, "", "",
- benchmarkTypeID, benchmarkChildTypeID, resourceSpecId)
- if err != nil {
- cloudBrainNewDataPrepare(ctx)
- ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form)
- return
- }
-
- ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
- }
-
- func BenchmarkDel(ctx *context.Context) {
- if err := deleteCloudbrainJob(ctx); err != nil {
- log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
- ctx.ServerError(err.Error(), err)
- return
- }
-
- var isAdminPage = ctx.Query("isadminpage")
- if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
- ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
- } else {
- ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
- }
- }
-
- func CloudBrainTrainJobNew(ctx *context.Context) {
- err := cloudBrainNewDataPrepare(ctx)
- if err != nil {
- ctx.ServerError("get new train-job info failed", err)
- return
- }
- ctx.HTML(http.StatusOK, tplCloudBrainTrainJobNew)
- }
-
- func getTrainJobCommand(form auth.CreateCloudBrainForm) (string, error) {
- var command string
- bootFile := form.BootFile
- params := form.Params
-
- if !strings.HasSuffix(bootFile, ".py") {
- log.Error("bootFile(%s) format error", bootFile)
- return command, errors.New("bootFile format error")
- }
-
- var parameters models.Parameters
- var param string
- if len(params) != 0 {
- err := json.Unmarshal([]byte(params), ¶meters)
- if err != nil {
- log.Error("Failed to Unmarshal params: %s (%v)", params, err)
- return command, err
- }
-
- for _, parameter := range parameters.Parameter {
- param += " --" + parameter.Label + "=" + parameter.Value
- }
- }
-
- command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile
-
- return command, nil
- }
-
- func CloudBrainTrainJobDel(ctx *context.Context) {
- var listType = ctx.Query("listType")
- if err := deleteCloudbrainJob(ctx); err != nil {
- log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
- ctx.ServerError(err.Error(), err)
- return
- }
-
- var isAdminPage = ctx.Query("isadminpage")
- if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
- ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
- } else {
- ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType)
- }
- }
-
- func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes {
- var lang = ctx.Locale.Language()
- if benchmarkTypesMap[lang] == nil {
- var val = i18n.Tr(lang, BENCHMARK_TYPE_CODE)
- //use config
- val = setting.BenchmarkTypes
- var tempType *models.BenchmarkTypes
- if err := json.Unmarshal([]byte(val), &tempType); err != nil {
- log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", val, err, ctx.Data["MsgID"])
- return &models.BenchmarkTypes{}
- }
- benchmarkTypesMap[lang] = tempType
- }
- return benchmarkTypesMap[lang]
- }
|