diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 7e736e19d7..294ab1e524 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2126,6 +2126,15 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { cb := &Cloudbrain{JobName: jobName} return getRepoCloudBrain(cb) } +func GetWaitOrRunFileNotebookByRepo(repoId int64, cloudbrainType int) (*Cloudbrain, error) { + cloudBrain := new(Cloudbrain) + has, err := x.In("status", JobWaiting, JobRunning, ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, + ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsDeleting, ModelArtsRestarting).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain) + if has { + return cloudBrain, err + } + return nil, err +} func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index c85f4b8cdf..827e2b7360 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -709,6 +709,7 @@ type GenerateModelArtsNotebookReq struct { ImageId string AutoStopDurationMs int64 + BranchName string Spec *models.Specification ModelName string diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index dcad1eb00b..bd7f848fc8 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,13 +1,9 @@ package modelarts import ( - "encoding/base64" "encoding/json" "errors" "fmt" - "io/ioutil" - "net/http" - "path" "strconv" "strings" @@ -239,6 +235,7 @@ func GenerateNotebook2(ctx *context.Context, req cloudbrain.GenerateModelArtsNot ComputeResource: models.NPUResource, Image: imageName, BootFile: req.BootFile, + BranchName: req.BranchName, Description: req.Description, CreatedUnix: createTime, UpdatedUnix: createTime, @@ -830,10 +827,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { task.FlavorCode = result.Flavor } - if oldStatus != task.Status && task.Status == string(models.ModelArtsRunning) && task.BootFile != "" { - uploadNoteBookFile(task, result) - - } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) @@ -844,81 +837,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { return nil } -func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) { - jupyterUrl := result.Url + "?token=" + result.Token - - cookies, xsrf := getCookiesAndCsrf(jupyterUrl) - if xsrf == "" { - log.Error("browser jupyterUrl failed:%v", task.DisplayJobName) - } else { - - codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath - fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile) - if err != nil { - log.Error("read jupyter file failed:%v", task.DisplayJobName, err) - } - - base64Content := base64.StdEncoding.EncodeToString(fileContents) - client := getRestyClient() - uploadUrl := getJupyterBaseUrl(result.Url) + "api/contents/" + path.Base(task.BootFile) - res, err := client.R(). - SetCookies(cookies). - SetHeader("X-XSRFToken", xsrf). - SetBody(map[string]interface{}{ - "type": "file", - "format": "base64", - "name": path.Base(task.BootFile), - "path": path.Base(task.BootFile), - "content": base64Content}). - Put(uploadUrl) - if err != nil { - log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) - } else if res.StatusCode() != http.StatusCreated { - log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) - } - - } - -} - -func getJupyterBaseUrl(url string) string { - jupyterUrlLength := len(url) - baseUrl := url[0 : jupyterUrlLength-len(path.Base(url))] - return baseUrl -} - -func getCookiesAndCsrf(jupyterUrl string) ([]*http.Cookie, string) { - log.Info("jupyter url:" + jupyterUrl) - var cookies []*http.Cookie - const retryTimes = 10 - for i := 0; i < retryTimes; i++ { - res, err := http.Get(jupyterUrl) - if err != nil { - log.Error("browser jupyterUrl failed.", err) - if i == retryTimes-1 { - return cookies, "" - } - - } else { - cookies = res.Cookies() - xsrf := "" - for _, cookie := range cookies { - if cookie.Name == "_xsrf" { - xsrf = cookie.Value - break - } - - } - if xsrf != "" { - return cookies, xsrf - } - - } - } - return cookies, "" - -} - func SyncTempStatusJob() { jobs, err := models.GetCloudBrainTempJobs() if err != nil { diff --git a/modules/modelarts_cd/modelarts.go b/modules/modelarts_cd/modelarts.go index bdc42002af..9d74c0919a 100755 --- a/modules/modelarts_cd/modelarts.go +++ b/modules/modelarts_cd/modelarts.go @@ -148,6 +148,7 @@ func GenerateNotebook(ctx *context.Context, req cloudbrain.GenerateModelArtsNote UpdatedUnix: createTime, Spec: req.Spec, BootFile: req.BootFile, + BranchName: req.BranchName, ModelName: req.ModelName, ModelVersion: req.ModelVersion, LabelName: req.LabelName, diff --git a/modules/notebook/contentManager.go b/modules/notebook/contentManager.go new file mode 100644 index 0000000000..823e596654 --- /dev/null +++ b/modules/notebook/contentManager.go @@ -0,0 +1,198 @@ +package notebook + +import ( + "crypto/tls" + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "path" + "strings" + + "github.com/go-resty/resty/v2" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/setting" + + "code.gitea.io/gitea/modules/log" +) + +var restyClient *resty.Client + +type NotebookApiResponse struct { + Name string `json:"name"` + Path string `json:"path"` +} + +type NotebookContent struct { + Url string + Path string + Cookies []*http.Cookie + Xsrf string + PathType string //file directory + Token string +} + +func (c *NotebookContent) IsNotebookFileCanBrowser() bool { + if c.Xsrf == "" { + c.SetCookiesAndCsrf() + } + if c.Xsrf == "" { + log.Warn("xsrf is empty, can not broswer url:" + c.Url) + return false + } + return c.IsNoteBookContentsExist() + +} + +func (c *NotebookContent) SetCookiesAndCsrf() { + log.Info("jupyter url:" + c.Url) + var cookies []*http.Cookie + const retryTimes = 10 + url := c.Url + if c.Token != "" { + url = c.Url + "?token=" + c.Token + } + for i := 0; i < retryTimes; i++ { + res, err := http.Get(url) + if err != nil { + log.Error("browser jupyterUrl failed.", err) + if i == retryTimes-1 { + c.Cookies = cookies + } + + } else { + cookies = res.Cookies() + xsrf := "" + for _, cookie := range cookies { + if cookie.Name == "_xsrf" { + xsrf = cookie.Value + if len(cookies) > 1 { + break + } + + } + + } + if xsrf != "" { + c.Cookies = cookies + c.Xsrf = xsrf + } + + } + } + c.Cookies = cookies + +} + +func (c *NotebookContent) IsNoteBookContentsExist() bool { + client := getRestyClient() + uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + "?type=" + c.PathType + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + Get(uploadUrl) + if err != nil { + log.Warn("browser url error:"+uploadUrl, err) + return false + } + return res.StatusCode() == http.StatusOK +} + +func (c *NotebookContent) UploadNoteBookFile(task *models.Cloudbrain) error { + + err := c.MakeNoteBookDir() + if err != nil { + return err + } + + codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath + fileContents, err := ioutil.ReadFile(codePath + "/" + c.Path) + if err != nil { + log.Error("read jupyter file failed:%v", task.DisplayJobName, err) + } + + base64Content := base64.StdEncoding.EncodeToString(fileContents) + client := getRestyClient() + uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + SetBody(map[string]interface{}{ + "type": "file", + "format": "base64", + "name": path.Base(c.Path), + "path": c.Path, + "content": base64Content}). + Put(uploadUrl) + if err != nil { + log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) + return err + } else if res.StatusCode() != http.StatusCreated { + log.Error("upload jupyter file failed:%v, status is %s", task.DisplayJobName, res.Status(), err) + return fmt.Errorf("status:", res.StatusCode()) + } + return nil +} + +/** + if c.Path is a/b/c.txt + makedir a/b + if c.Path is a/b/c + makedir a/b +*/ +func (c *NotebookContent) MakeNoteBookDir() error { + filePaths := strings.Split(c.Path, "/") + + for i := 0; i < len(filePaths)-1; i++ { + cTemp := &NotebookContent{ + Url: c.Url, + Cookies: c.Cookies, + Path: path.Join(filePaths[0 : i+1]...), + PathType: "directory", + Xsrf: c.Xsrf, + } + if !cTemp.IsNoteBookContentsExist() { + + createTempDirUrl := getJupyterBaseUrl(cTemp.Url) + "api/contents/" + cTemp.Path + client := getRestyClient() + var jobResult NotebookApiResponse + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + SetBody(map[string]interface{}{ + "type": cTemp.PathType, + "path": cTemp.Path, + }).SetResult(&jobResult). + Put(createTempDirUrl) + if err != nil { + return err + } + if res.StatusCode() != http.StatusCreated { + return fmt.Errorf("status code:" + res.Status()) + } + + } + + } + return nil +} + +func getJupyterBaseUrl(url string) string { + jupyterUrlLength := len(url) + baseUrl := url + if strings.HasSuffix(url, "lab") { + baseUrl = url[0 : jupyterUrlLength-len(path.Base(url))] + } + + return baseUrl +} + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + } + return restyClient +} diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index 9ea5601c9c..e4403ed049 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -47,6 +47,7 @@ type CreateFileNotebookJobOption struct { BranchName string `json:"branch_name" binding:"Required"` OwnerName string `json:"owner_name" binding:"Required"` ProjectName string `json:"project_name" binding:"Required"` + JobId string `json:"job_id"` } type Cloudbrain struct { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 70c5a34089..f05ea57997 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1062,7 +1062,7 @@ model_rename=Duplicate model name, please modify model name. notebook_file_not_exist=Notebook file does not exist. notebook_select_wrong=Please select a Notebook(.ipynb) file first. notebook_file_no_right=You have no right to access the Notebook(.ipynb) file. -debug_again_fail=Fail to restart debug task, please try again later. +notebook_repo_conflict=The files in different branches of the same repository can not run together. date=Date repo_add=Project Increment diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index a790919329..05dccb5929 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1061,7 +1061,7 @@ model_rename=模型名称重复，请修改模型名称 notebook_file_not_exist=Notebook文件不存在。 notebook_select_wrong=请先选择Notebook（.ipynb）文件。 notebook_file_no_right=您没有这个Notebook文件的读权限。 -debug_again_fail=再次调试失败，请稍后再试。 +notebook_repo_conflict=同一个仓库的不同分支文件不能同时运行。 date=日期 repo_add=新增项目 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 4f86098d92..b5ba895be4 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -745,7 +745,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/file_notebook", func() { m.Get("", repo.GetFileNoteBookInfo) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) - + m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) }) m.Group("/repos", func() { diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 8054437887..250a084232 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -110,6 +110,9 @@ func GeneralCloudBrainJobStop(ctx *context.APIContext) { func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { cloudbrainTask.FileNotebookCreate(ctx.Context, option) } +func FileNoteBookStatus(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { + cloudbrainTask.FileNotebookStatus(ctx.Context, option) +} func GetFileNoteBookInfo(ctx *context.APIContext) { //image description spec description waiting count diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 3c723044ce..1076911e06 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -911,10 +911,13 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName - if task.BootFile != "" { - ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) + if ctx.QueryTrim("file") != "" { + ctx.Redirect(getFileUrl(debugUrl, ctx.QueryTrim("file"))) } else { + if task.BootFile != "" { + go cloudbrainTask.UploadNotebookFiles(task) + } ctx.Redirect(debugUrl) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index bfe051c3a1..86f3b37cdb 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -439,9 +439,13 @@ func NotebookDebug2(ctx *context.Context) { ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil) return } - if task.BootFile != "" { - ctx.Redirect(getFileUrl(result.Url, task.BootFile) + "?token=" + result.Token) + + if ctx.QueryTrim("file") != "" { + ctx.Redirect(getFileUrl(result.Url, ctx.QueryTrim("file")) + "?token=" + result.Token) } else { + if task.BootFile != "" { + go cloudbrainTask.UploadNotebookFiles(task) + } ctx.Redirect(result.Url + "?token=" + result.Token) } @@ -463,7 +467,7 @@ func getFileUrl(url string, filename string) string { } } - return url + middle + path.Base(filename) + return url + middle + filename } func NotebookRestart(ctx *context.Context) { @@ -630,7 +634,7 @@ func NotebookStop(ctx *context.Context) { if task.Status != string(models.ModelArtsRunning) { log.Error("the job(%s) is not running", task.JobName, ctx.Data["MsgID"]) resultCode = "-1" - errorMsg = "the job is not running" + errorMsg = ctx.Tr("cloudbrain.Already_stopped") break } diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index cc95635207..3526f6549f 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -4,6 +4,9 @@ import ( "fmt" "net/http" "path" + "strings" + + "code.gitea.io/gitea/modules/notebook" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/modelarts_cd" @@ -29,6 +32,9 @@ import ( ) const NoteBookExtension = ".ipynb" +const CPUType = 0 +const GPUType = 1 +const NPUType = 2 func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { @@ -66,7 +72,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp } //create repo if not exist - repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) + repo, _ := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) if repo == nil { repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ Name: setting.FileNoteBook.ProjectName, @@ -80,19 +86,222 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp AutoInit: true, DefaultBranch: "master", }) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) + return + } + } else { + + noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type)) + if noteBook != nil { + + if isRepoConfilcts(option, noteBook) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict"))) + return + } + + if isNotebookSpecMath(option, noteBook) { + if !isRepoMatch(option, noteBook) { + err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + } + if !isRepoFileMatch(option, noteBook) { + noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) + noteBook.BranchName += ";" + option.BranchName + noteBook.Description += ";" + getDescription(option) + + err := models.UpdateJob(noteBook) + if err != nil { + log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + } + + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: noteBook.JobID, + }) + return + } + + } + } + + if option.Type <= GPUType { + cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) + } else { + modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) + } + +} +func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) { + if ctx.Written() { + return } + + if path.Ext(option.File) != NoteBookExtension { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong"))) + return + } + + isNotebookFileExist, _ := isNoteBookFileExist(ctx, option) + if !isNotebookFileExist { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + task, err := models.GetCloudbrainByJobID(option.JobId) if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) + log.Error("job not found:"+option.JobId, err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Job id may not be right. can not find job.")) return } - if option.Type <= 1 { - cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) + if task.BootFile == "" || task.Status != string(models.ModelArtsRunning) { + log.Warn("Boot file is empty or status is running. ") + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Boot file is empty or status is running.")) + return + } + if !isRepoFileMatch(option, task) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("can not math repo file.")) + return + } + debugBaseUrl, token, err := getBaseUrlAndToken(task) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) { + ctx.JSON(http.StatusOK, models.BaseOKMessageApi) } else { - modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed.")) + + } + +} + +func getBaseUrlAndToken(task *models.Cloudbrain) (string, string, error) { + var debugBaseUrl string + var token string + if task.Type == models.TypeCloudBrainOne { + debugBaseUrl = setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + "/lab" + + } else { + var result *models.GetNotebook2Result + var err error + if task.Type == models.TypeCloudBrainTwo { + result, err = modelarts.GetNotebook2(task.JobID) + } else if task.Type == models.TypeCDCenter { + result, err = modelarts_cd.GetNotebook(task.JobID) + } + if err != nil || result == nil || result.Status != string(models.ModelArtsRunning) || result.Url == "" { + log.Error("notebook job not found:"+task.JobID, err) + return "", "", fmt.Errorf("can not get job or job is invalid.") + } + + debugBaseUrl = result.Url + token = result.Token + } + return debugBaseUrl, token, nil +} + +func uploadNotebookFileIfCannotBroswer(debugBaseUrl string, bootFile string, task *models.Cloudbrain, token string) bool { + c := ¬ebook.NotebookContent{ + Url: debugBaseUrl, + Path: bootFile, + PathType: "file", + Token: token, + } + if c.IsNotebookFileCanBrowser() { + return true + } else { + c.SetCookiesAndCsrf() + c.UploadNoteBookFile(task) + return c.IsNotebookFileCanBrowser() + } + +} + +func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + if option.Type == NPUType || option.Type == CPUType { + return true + } + spec, err := models.GetCloudbrainSpecByID(book.ID) + if err != nil { + log.Warn("can not get spec ", err) + return false + } + return spec.AccCardsNum > 0 +} + +func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + branches := strings.Split(book.BranchName, ";") + + for i, bootFile := range bootFiles { + splits := strings.Split(bootFile, "/") + if len(splits) >= 3 { + if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName { + return true + } + } + } + + return false } +func isRepoMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + + for _, bootFile := range bootFiles { + splits := strings.Split(bootFile, "/") + if len(splits) >= 3 { + if splits[0] == option.OwnerName && splits[1] == option.ProjectName { + return true + } + } + } + return false + +} + +func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + branches := strings.Split(book.BranchName, ";") + + for i, bootFile := range bootFiles { + if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile { + return true + } + } + + return false + +} +func UploadNotebookFiles(task *models.Cloudbrain) { + if task.Status == string(models.JobRunning) && task.BootFile != "" { + + debugBaseUrl, token, err := getBaseUrlAndToken(task) + if err != nil { + log.Error("can not get base url:", err) + return + } + bootFiles := strings.Split(task.BootFile, ";") + + for _, bootFile := range bootFiles { + uploadNotebookFileIfCannotBroswer(debugBaseUrl, bootFile, task, token) + } + + } +} + func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) @@ -131,17 +340,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK,models.BaseMessageApi{ - Code: 2, + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 2, Message: ctx.Tr("repo.cloudbrain.morethanonejob"), }) return } } - errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) - if errStr != "" { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return } command := cloudbrain.GetCloudbrainDebugCommand() @@ -185,7 +395,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot JobType: jobType, Description: getDescription(option), BranchName: option.BranchName, - BootFile: option.File, + BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), Params: "{\"parameter\":[]}", CommitID: "", BenchmarkTypeID: 0, @@ -206,8 +416,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } -func getCodePath(jobName string) string { - return setting.JobPath + jobName + cloudbrain.CodeMountPath +func getCloudbrainType(optionType int) int { + if optionType < 1 { + return models.TypeCloudBrainOne + } + if setting.ModelartsCD.Enabled { + return models.TypeCDCenter + } + return models.TypeCloudBrainTwo +} + +func getCodePath(jobName string, repo *models.Repository) string { + return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name } func getDescription(option api.CreateFileNotebookJobOption) string { @@ -237,8 +457,8 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK,models.BaseMessageApi{ - Code: 2, + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 2, Message: ctx.Tr("repo.cloudbrain.morethanonejob"), }) return @@ -260,7 +480,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } } - err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) + err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return @@ -297,8 +517,9 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote Description: getDescription(option), ImageId: setting.FileNoteBook.ImageIdNPU, Spec: spec, - BootFile: "", + BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), AutoStopDurationMs: modelarts.AutoStopDurationMs / 4, + BranchName: option.BranchName, } if setting.ModelartsCD.Enabled { @@ -347,17 +568,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO return true, nil } -func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { - err := downloadCode(repo, codePath, branchName) - if err != nil { - return "cloudbrain.load_code_failed" - } - - err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/") - if err != nil { - return "cloudbrain.load_code_failed" - } - return "" +func getBootFile(filePath string, ownerName string, projectName string) string { + return ownerName + "/" + projectName + "/" + filePath } func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { diff --git a/web_src/vuepages/apis/modules/notobook.js b/web_src/vuepages/apis/modules/notobook.js index 4c59a253bc..1dd35e3378 100644 --- a/web_src/vuepages/apis/modules/notobook.js +++ b/web_src/vuepages/apis/modules/notobook.js @@ -29,7 +29,7 @@ export const getCb1Notebook = (path,jobid) => { }); }; -// Notebook获取云脑I调试任务状态 +// Notebook获取云脑II调试任务状态 export const getCb2Notebook = (path,jobid) => { return service({ url: `/api/v1/${path}/modelarts/notebook/${jobid}`, @@ -37,7 +37,16 @@ export const getCb2Notebook = (path,jobid) => { params: {}, }); }; - +// Notebook查询文件在环境中是否已准备好 +// type, file, branch_name, owner_name, project_name,job_id +export const getFileInfoNotebook = (data) => { + return service({ + url: "/api/v1/file_notebook/status", + method: "post", + data, + params: {}, + }); +}; export const stopNotebook = (url) => { return service({ url: url, diff --git a/web_src/vuepages/pages/notebook/debug/index.vue b/web_src/vuepages/pages/notebook/debug/index.vue index 20af220f57..a4c93529a3 100644 --- a/web_src/vuepages/pages/notebook/debug/index.vue +++ b/web_src/vuepages/pages/notebook/debug/index.vue @@ -178,9 +178,9 @@