|
- package repository
-
- import (
- "bytes"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "io/ioutil"
- "net/http"
-
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/setting"
- )
-
- //输入elk的json结构begin
- type InputInfo struct {
- Batch []Batch `json:"batch"`
- }
- type Fields struct {
- Field string `json:"field"`
- Format string `json:"format"`
- }
- type MatchPhrase struct {
- Message string `json:"message,omitempty"`
- TagName string `json:"tagName.keyword,omitempty"`
- }
- type Should struct {
- MatchPhrase MatchPhrase `json:"match_phrase"`
- }
- type Bool struct {
- Should []Should `json:"should"`
- MinimumShouldMatch int `json:"minimum_should_match"`
- }
- type Timestamptest struct {
- // Gte time.Time `json:"gte"`
- Gte string `json:"gte"`
- Lte string `json:"lte"`
- Format string `json:"format"`
- }
- type Range struct {
- Timestamptest Timestamptest `json:"@timestamptest"`
- }
-
- type FilterMatchPhrase struct {
- UserName string `json:"userName.keyword,omitempty"`
- ProjectName string `json:"projectName.keyword,omitempty"`
- TagName string `json:"tagName.keyword,omitempty"`
- }
-
- type Filter struct {
- Bool *Bool `json:"bool,omitempty"`
- Range *Range `json:"range,omitempty"`
- FilterMatchPhrase *FilterMatchPhrase `json:"match_phrase,omitempty"`
- }
- type MustNotMatchPhrase struct {
- ProjectName string `json:"projectName"`
- }
- type MustNot struct {
- MustNotMatchPhrase MustNotMatchPhrase `json:"match_phrase"`
- }
- type BoolIn struct {
- Filter []Filter `json:"filter"`
- MustNot []MustNot `json:"must_not"`
- }
- type Query struct {
- BoolIn BoolIn `json:"bool"`
- }
- type Body struct {
- Size int `json:"size"`
- Fields []Fields `json:"fields"`
- Query Query `json:"query"`
- }
- type Params struct {
- Index string `json:"index"`
- Body Body `json:"body"`
- }
- type Request struct {
- Params Params `json:"params"`
- }
- type Batch struct {
- Request Request `json:"request"`
- }
-
- //输入elk的json结构end
-
- //elk输出的json结构begin
- type Hits struct {
- Total int `json:"total"`
- }
- type RawResponse struct {
- Hits Hits `json:"hits"`
- }
- type Result struct {
- RawResponse RawResponse `json:"rawResponse"`
- Loaded int `json:"loaded"`
- }
- type ResultInfo struct {
- Id int `json:"id"`
- Result Result `json:"result"`
- }
-
- //elk输出的json结构end
-
- //处理返回的elk数据,只保留totalView,即访问量;loaded是分片载入次数,用来判断返回的数据是否准确
- func GetResultFromElk(resultInfo ResultInfo, jsonStr []byte) (loaded int, totalView int, err error) {
- ElkBase64Init := setting.ElkUser + ":" + setting.ElkPassword
- ElkBase64 := base64.StdEncoding.EncodeToString([]byte(ElkBase64Init))
- BasicElkBase64 := "Basic" + " " + ElkBase64
- url := setting.ElkUrl
- req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("kbn-version", "7.13.2")
- req.Header.Set("Authorization", BasicElkBase64)
-
- client := &http.Client{}
- resp, err := client.Do(req)
- if err != nil {
- // panic(err)
- return 0, 0, err
- }
- defer resp.Body.Close()
- if resp.StatusCode != 200 {
- log.Error("ConnectToElk failed:%s", resp.Status)
- return 0, 0, fmt.Errorf("ConnectToElk failed:%s", resp.Status)
- }
- body, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return 0, 0, err
- }
- err = json.Unmarshal([]byte(string(body)), &resultInfo)
- if err != nil {
- log.Error("Get resultJson failed", err)
- return 0, 0, err
- }
-
- return resultInfo.Result.Loaded, resultInfo.Result.RawResponse.Hits.Total, err
- }
-
- //初始化传给elk的数据结构,给定用户名和项目名,查询的起止时间,返回初始化后的结构
- func ProjectViewInit(User string, Project string, Gte string, Lte string) (projectViewInit InputInfo) {
- var inputStruct InputInfo
- inputStruct.Batch = make([]Batch, 1)
- inputStruct.Batch[0].Request.Params.Index = setting.Index
- inputStruct.Batch[0].Request.Params.Body.Size = 0
- inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1)
- inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField
- inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 4)
- //限定查询时间
- var timeRange Range
- timeRange.Timestamptest.Gte = Gte
- timeRange.Timestamptest.Lte = Lte
- timeRange.Timestamptest.Format = "strict_date_optional_time"
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Range = &timeRange
- //限定用户
- var userName FilterMatchPhrase
- userName.UserName = User
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].FilterMatchPhrase = &userName
- //限定项目
- var projectName FilterMatchPhrase
- projectName.ProjectName = Project
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].FilterMatchPhrase = &projectName
- //限定页面
- var bool Bool
- bool.Should = make([]Should, len(setting.PROJECT_LIMIT_PAGES))
- for i, pageName := range setting.PROJECT_LIMIT_PAGES {
- bool.Should[i].MatchPhrase.TagName = pageName
- }
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[3].Bool = &bool
- return inputStruct
- }
-
- //初始化传给elk的数据结构,给定查询信息和非项目名,查询的起止时间,返回初始化后的结构
- func AllProjectViewInit(MessageInfo string, NotProject string, Gte string, Lte string) (allProjectViewInit InputInfo) {
- var inputStruct InputInfo
- inputStruct.Batch = make([]Batch, 1)
- inputStruct.Batch[0].Request.Params.Index = setting.Index
- inputStruct.Batch[0].Request.Params.Body.Size = 0
- inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1)
- inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField
- inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 2)
- //限定message
- var bool Bool
- bool.Should = make([]Should, 1)
- bool.Should[0].MatchPhrase.Message = MessageInfo
- bool.MinimumShouldMatch = 1
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Bool = &bool
- //限定查询时间
- var timeRange Range
- timeRange.Timestamptest.Gte = Gte
- timeRange.Timestamptest.Lte = Lte
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].Range = &timeRange
- //限定非项目
- // var boolIn BoolIn
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.MustNot = make([]MustNot, 1)
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.MustNot[0].MustNotMatchPhrase.ProjectName = NotProject
- return inputStruct
- }
-
- //初始化传给elk的数据结构,给定查询信息和tagName,查询的起止时间,返回初始化后的结构
- func TagNameInit(MessageInfo string, Tagname string, Gte string, Lte string) (projectViewInit InputInfo) {
- var inputStruct InputInfo
- inputStruct.Batch = make([]Batch, 1)
- inputStruct.Batch[0].Request.Params.Index = setting.Index
- inputStruct.Batch[0].Request.Params.Body.Size = 0
- inputStruct.Batch[0].Request.Params.Body.Fields = make([]Fields, 1)
- inputStruct.Batch[0].Request.Params.Body.Fields[0].Field = setting.TimeField
- inputStruct.Batch[0].Request.Params.Body.Fields[0].Format = setting.ElkTimeFormat
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter = make([]Filter, 3)
- //限定message
- var bool Bool
- bool.Should = make([]Should, 1)
- bool.Should[0].MatchPhrase.Message = MessageInfo
- bool.MinimumShouldMatch = 1
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[0].Bool = &bool
- //限定tagName
- var tagName FilterMatchPhrase
- tagName.TagName = Tagname
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[1].FilterMatchPhrase = &tagName
- //限定查询时间
- var timeRange Range
- timeRange.Timestamptest.Gte = Gte
- timeRange.Timestamptest.Lte = Lte
- inputStruct.Batch[0].Request.Params.Body.Query.BoolIn.Filter[2].Range = &timeRange
- return inputStruct
- }
-
- //向elk发送请求,将获取的结果只保留访问量,输入是初始化后的数据结构,返回访问量
- func ViewInfo(viewInfo InputInfo) (totalView int, err error) {
- jsons, err := json.Marshal(viewInfo)
- if err != nil {
- log.Error("jsons failed", err)
- return 0, err
- }
- var jsonStr = []byte(jsons)
- var resultInfo ResultInfo
- loaded, totalView, err := GetResultFromElk(resultInfo, jsonStr)
-
- time := 0
- for {
- if loaded == 0 {
- loaded_next, totalView, err := GetResultFromElk(resultInfo, jsonStr)
- time++
- if loaded_next != 0 && time < 100 {
- return totalView, err
- }
- if time > 100 {
- break
- }
- } else {
- break
- }
- }
- return totalView, err
- }
-
- // @title AppointProjectView
- // @description 获取指定用户和项目的访问量
- // @param User string "用户名"
- // @param Project string "项目名"
- // @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339)
- // @param Lte string "结束时间" 如time.Now().Format(time.RFC3339)
- // @return totalView int "访问量"
- func AppointProjectView(User string, Project string, Gte string, Lte string) (totalView int, err error) {
- ProjectViewInitInfo := ProjectViewInit(User, Project, Gte, Lte)
- ProjectTotalView, err := ViewInfo(ProjectViewInitInfo)
- return ProjectTotalView, err
- }
-
- //统计项目相关页面的访问量
- type ProjectInfo struct {
- /* 统计所有项目中该页面的浏览情况,不需要区分项目。以aiforge项目为例 */
- //地址:https://git.openi.org.cn/OpenI/aiforge/datasets?type=0
- Project_dataset_type_0 int
- //地址:https://git.openi.org.cn/OpenI/aiforge/datasets?type=1
- Project_dataset_type_1 int
- //地址:https://git.openi.org.cn/OpenI/aiforge/issues
- Project_issues int
- //地址:https://git.openi.org.cn/OpenI/aiforge/labels
- Project_labels int
- //地址:https://git.openi.org.cn/OpenI/aiforge/milestones
- Project_milestones int
- //地址:https://git.openi.org.cn/OpenI/aiforge/pulls
- Project_pulls int
- //地址:https://git.openi.org.cn/OpenI/aiforge/release
- Project_release int
- //地址:https://git.openi.org.cn/OpenI/aiforge/wiki
- Project_wiki int
- //地址:https://git.openi.org.cn/OpenI/aiforge/activity
- Project_activity int
- //地址:https://git.openi.org.cn/OpenI/aiforge/cloudbrain
- Project_cloudbrain int
- //地址:https://git.openi.org.cn/OpenI/aiforge/modelarts
- Project_modelarts int
- //地址:https://git.openi.org.cn/OpenI/aiforge/blockchain
- Project_blockchain int
- //地址:https://git.openi.org.cn/OpenI/aiforge/watchers
- Project_watchers int
- //地址:https://git.openi.org.cn/OpenI/aiforge/stars
- Project_stars int
- //地址:https://git.openi.org.cn/OpenI/aiforge/forks
- Project_forks int
- }
-
- type ErrorInfo struct {
- Project_dataset_type_0 error
- Project_dataset_type_1 error
- Project_issues error
- Project_labels error
- Project_milestones error
- Project_pulls error
- Project_release error
- Project_wiki error
- Project_activity error
- Project_cloudbrain error
- Project_modelarts error
- Project_blockchain error
- Project_watchers error
- Project_stars error
- Project_forks error
- }
-
- // @title AllProjectView
- // @description 获取指定用户和项目的访问量
- // @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339)
- // @param Lte string "结束时间"
- // @return projectInfo ProjectInfo "统计所有项目中页面的浏览情况,不需要区分项目"
- func AllProjectView(Gte string, Lte string) (projectViewInfo ProjectInfo, errorInfo ErrorInfo) {
- projectViewInfo.Project_dataset_type_0, errorInfo.Project_dataset_type_0 = ViewInfo(AllProjectViewInit("/datasets?type=0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_dataset_type_1, errorInfo.Project_dataset_type_1 = ViewInfo(AllProjectViewInit("/datasets?type=1", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_issues, errorInfo.Project_issues = ViewInfo(AllProjectViewInit("/issues HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_labels, errorInfo.Project_labels = ViewInfo(TagNameInit("/labels HTTP/2.0", "labels", Gte, Lte))
- projectViewInfo.Project_milestones, errorInfo.Project_milestones = ViewInfo(AllProjectViewInit("/milestones HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_pulls, errorInfo.Project_pulls = ViewInfo(AllProjectViewInit("/pulls HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_release, errorInfo.Project_release = ViewInfo(AllProjectViewInit("/release HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_wiki, errorInfo.Project_wiki = ViewInfo(AllProjectViewInit("/wiki HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_activity, errorInfo.Project_activity = ViewInfo(AllProjectViewInit("/activity HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_cloudbrain, errorInfo.Project_cloudbrain = ViewInfo(AllProjectViewInit("/cloudbrain HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_modelarts, errorInfo.Project_modelarts = ViewInfo(AllProjectViewInit("/modelarts HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_blockchain, errorInfo.Project_blockchain = ViewInfo(AllProjectViewInit("/blockchain HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_watchers, errorInfo.Project_watchers = ViewInfo(AllProjectViewInit("/watchers HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_stars, errorInfo.Project_stars = ViewInfo(AllProjectViewInit("/stars HTTP/2.0", "%{[request][2]}", Gte, Lte))
- projectViewInfo.Project_forks, errorInfo.Project_forks = ViewInfo(AllProjectViewInit("/forks HTTP/2.0", "%{[request][2]}", Gte, Lte))
- return projectViewInfo, errorInfo
- }
|