Browse Source

Add a storage layer for attachments

tags/12
Lunny Xiao yan <sxty32@gmail.com> 1 year ago
parent
commit
3508716367
100 changed files with 16319 additions and 67 deletions
  1. +68
    -0
      cmd/migrate_storage.go
  2. +2
    -0
      go.mod
  3. +4
    -0
      go.sum
  4. +1
    -4
      integrations/attachment_test.go
  5. +1
    -0
      main.go
  6. +13
    -0
      models/admin.go
  7. +32
    -33
      models/attachment.go
  8. +2
    -3
      models/migrations/v112.go
  9. +11
    -6
      models/migrations/v96.go
  10. +5
    -4
      models/repo.go
  11. +2
    -13
      modules/migrations/gitea.go
  12. +62
    -0
      modules/storage/local.go
  13. +59
    -0
      modules/storage/minio.go
  14. +42
    -0
      modules/storage/storage.go
  15. +4
    -0
      routers/init.go
  16. +2
    -2
      routers/repo/attachment.go
  17. +2
    -2
      services/release/release.go
  18. +6
    -0
      vendor/github.com/go-ini/ini/.gitignore
  19. +191
    -0
      vendor/github.com/go-ini/ini/LICENSE
  20. +15
    -0
      vendor/github.com/go-ini/ini/Makefile
  21. +43
    -0
      vendor/github.com/go-ini/ini/README.md
  22. +9
    -0
      vendor/github.com/go-ini/ini/codecov.yml
  23. +76
    -0
      vendor/github.com/go-ini/ini/data_source.go
  24. +25
    -0
      vendor/github.com/go-ini/ini/deprecated.go
  25. +34
    -0
      vendor/github.com/go-ini/ini/error.go
  26. +509
    -0
      vendor/github.com/go-ini/ini/file.go
  27. +24
    -0
      vendor/github.com/go-ini/ini/helper.go
  28. +168
    -0
      vendor/github.com/go-ini/ini/ini.go
  29. +829
    -0
      vendor/github.com/go-ini/ini/key.go
  30. +535
    -0
      vendor/github.com/go-ini/ini/parser.go
  31. +256
    -0
      vendor/github.com/go-ini/ini/section.go
  32. +724
    -0
      vendor/github.com/go-ini/ini/struct.go
  33. +3
    -0
      vendor/github.com/minio/minio-go/.gitignore
  34. +28
    -0
      vendor/github.com/minio/minio-go/.travis.yml
  35. +23
    -0
      vendor/github.com/minio/minio-go/CONTRIBUTING.md
  36. +202
    -0
      vendor/github.com/minio/minio-go/LICENSE
  37. +35
    -0
      vendor/github.com/minio/minio-go/MAINTAINERS.md
  38. +15
    -0
      vendor/github.com/minio/minio-go/Makefile
  39. +2
    -0
      vendor/github.com/minio/minio-go/NOTICE
  40. +239
    -0
      vendor/github.com/minio/minio-go/README.md
  41. +245
    -0
      vendor/github.com/minio/minio-go/README_zh_CN.md
  42. +565
    -0
      vendor/github.com/minio/minio-go/api-compose-object.go
  43. +84
    -0
      vendor/github.com/minio/minio-go/api-datatypes.go
  44. +282
    -0
      vendor/github.com/minio/minio-go/api-error-response.go
  45. +77
    -0
      vendor/github.com/minio/minio-go/api-get-lifecycle.go
  46. +136
    -0
      vendor/github.com/minio/minio-go/api-get-object-acl.go
  47. +26
    -0
      vendor/github.com/minio/minio-go/api-get-object-context.go
  48. +125
    -0
      vendor/github.com/minio/minio-go/api-get-object-file.go
  49. +659
    -0
      vendor/github.com/minio/minio-go/api-get-object.go
  50. +128
    -0
      vendor/github.com/minio/minio-go/api-get-options.go
  51. +78
    -0
      vendor/github.com/minio/minio-go/api-get-policy.go
  52. +715
    -0
      vendor/github.com/minio/minio-go/api-list.go
  53. +228
    -0
      vendor/github.com/minio/minio-go/api-notification.go
  54. +215
    -0
      vendor/github.com/minio/minio-go/api-presigned.go
  55. +306
    -0
      vendor/github.com/minio/minio-go/api-put-bucket.go
  56. +111
    -0
      vendor/github.com/minio/minio-go/api-put-object-common.go
  57. +33
    -0
      vendor/github.com/minio/minio-go/api-put-object-context.go
  58. +83
    -0
      vendor/github.com/minio/minio-go/api-put-object-copy.go
  59. +64
    -0
      vendor/github.com/minio/minio-go/api-put-object-file-context.go
  60. +27
    -0
      vendor/github.com/minio/minio-go/api-put-object-file.go
  61. +372
    -0
      vendor/github.com/minio/minio-go/api-put-object-multipart.go
  62. +417
    -0
      vendor/github.com/minio/minio-go/api-put-object-streaming.go
  63. +267
    -0
      vendor/github.com/minio/minio-go/api-put-object.go
  64. +303
    -0
      vendor/github.com/minio/minio-go/api-remove.go
  65. +245
    -0
      vendor/github.com/minio/minio-go/api-s3-datatypes.go
  66. +532
    -0
      vendor/github.com/minio/minio-go/api-select.go
  67. +185
    -0
      vendor/github.com/minio/minio-go/api-stat.go
  68. +898
    -0
      vendor/github.com/minio/minio-go/api.go
  69. +39
    -0
      vendor/github.com/minio/minio-go/appveyor.yml
  70. +221
    -0
      vendor/github.com/minio/minio-go/bucket-cache.go
  71. +273
    -0
      vendor/github.com/minio/minio-go/bucket-notification.go
  72. +62
    -0
      vendor/github.com/minio/minio-go/constants.go
  73. +153
    -0
      vendor/github.com/minio/minio-go/core.go
  74. +71
    -0
      vendor/github.com/minio/minio-go/hook-reader.go
  75. +89
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/chain.go
  76. +17
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/config.json.sample
  77. +175
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/credentials.go
  78. +12
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/credentials.sample
  79. +62
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/doc.go
  80. +71
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/env_aws.go
  81. +62
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/env_minio.go
  82. +120
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/file_aws_credentials.go
  83. +133
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go
  84. +250
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/iam_aws.go
  85. +77
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/signature-type.go
  86. +67
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/static.go
  87. +173
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/sts_client_grants.go
  88. +169
    -0
      vendor/github.com/minio/minio-go/pkg/credentials/sts_web_identity.go
  89. +195
    -0
      vendor/github.com/minio/minio-go/pkg/encrypt/server-side.go
  90. +306
    -0
      vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go
  91. +316
    -0
      vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go
  92. +315
    -0
      vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go
  93. +49
    -0
      vendor/github.com/minio/minio-go/pkg/s3signer/utils.go
  94. +331
    -0
      vendor/github.com/minio/minio-go/pkg/s3utils/utils.go
  95. +197
    -0
      vendor/github.com/minio/minio-go/pkg/set/stringset.go
  96. +270
    -0
      vendor/github.com/minio/minio-go/post-policy.go
  97. +69
    -0
      vendor/github.com/minio/minio-go/retry-continous.go
  98. +153
    -0
      vendor/github.com/minio/minio-go/retry.go
  99. +52
    -0
      vendor/github.com/minio/minio-go/s3-endpoints.go
  100. +61
    -0
      vendor/github.com/minio/minio-go/s3-error.go

+ 68
- 0
cmd/migrate_storage.go View File

@@ -0,0 +1,68 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
"context"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"

"github.com/urfave/cli"
)

// CmdMigrateStorage represents the available migrate storage sub-command.
var CmdMigrateStorage = cli.Command{
Name: "migrate-storage",
Usage: "Migrate the storage",
Description: "This is a command for migrating storage.",
Action: runMigrateStorage,
}

func migrateAttachments(dstStorage storage.ObjectStorage) error {
return models.IterateAttachment(func(attach *models.Attachment) error {
_, err := storage.Copy(dstStorage, attach.UUID, storage.Attachments, attach.RelativePath())
return err
})
}

func runMigrateStorage(ctx *cli.Context) error {
if err := initDB(); err != nil {
return err
}

log.Trace("AppPath: %s", setting.AppPath)
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
log.Trace("Custom path: %s", setting.CustomPath)
log.Trace("Log path: %s", setting.LogRootPath)
setting.InitDBConfig()

if err := models.NewEngine(context.Background(), migrations.Migrate); err != nil {
log.Fatal("Failed to initialize ORM engine: %v", err)
return err
}

tp := ctx.String("type")

// TODO: init setting

if err := storage.Init(); err != nil {
return err
}

switch tp {
case "attachments":
dstStorage, err := storage.NewLocalStorage(ctx.String("dst"))
if err != nil {
return err
}
return migrateAttachments(dstStorage)
}

return nil
}

+ 2
- 0
go.mod View File

@@ -40,6 +40,7 @@ require (
github.com/go-enry/go-enry/v2 v2.3.0
github.com/go-git/go-billy/v5 v5.0.0
github.com/go-git/go-git/v5 v5.0.0
github.com/go-ini/ini v1.56.0 // indirect
github.com/go-openapi/jsonreference v0.19.3 // indirect
github.com/go-redis/redis v6.15.2+incompatible
github.com/go-sql-driver/mysql v1.4.1
@@ -72,6 +73,7 @@ require (
github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81
github.com/mgechev/revive v1.0.2
github.com/microcosm-cc/bluemonday v1.0.3-0.20191119130333-0a75d7616912
github.com/minio/minio-go v6.0.14+incompatible
github.com/mitchellh/go-homedir v1.1.0
github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5


+ 4
- 0
go.sum View File

@@ -208,6 +208,8 @@ github.com/go-git/go-git-fixtures/v4 v4.0.1 h1:q+IFMfLx200Q3scvt2hN79JsEzy4AmBTp
github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
github.com/go-git/go-git/v5 v5.0.0 h1:k5RWPm4iJwYtfWoxIJy4wJX9ON7ihPeZZYC1fLYDnpg=
github.com/go-git/go-git/v5 v5.0.0/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmClfZwtUVA=
github.com/go-ini/ini v1.56.0 h1:6HjxSjqdmgnujDPhlzR4a44lxK3w03WPN8te0SoUSeM=
github.com/go-ini/ini v1.56.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@@ -469,6 +471,8 @@ github.com/mgechev/revive v1.0.2 h1:v0NxxQ7fSFz/u1NQydPo6EGdq7va0J1BtsZmae6kzUg=
github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo=
github.com/microcosm-cc/bluemonday v1.0.3-0.20191119130333-0a75d7616912 h1:hJde9rA24hlTcAYSwJoXpDUyGtfKQ/jsofw+WaDqGrI=
github.com/microcosm-cc/bluemonday v1.0.3-0.20191119130333-0a75d7616912/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w=
github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=


+ 1
- 4
integrations/attachment_test.go View File

@@ -123,10 +123,7 @@ func TestGetAttachment(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
//Write empty file to be available for response
if tc.createFile {
localPath := models.AttachmentLocalPath(tc.uuid)
err := os.MkdirAll(path.Dir(localPath), os.ModePerm)
assert.NoError(t, err)
err = ioutil.WriteFile(localPath, []byte("hello world"), 0644)
err = SaveAttachment(tc.uuid, strings.NewReader("hello world"))
assert.NoError(t, err)
}
//Actual test


+ 1
- 0
main.go View File

@@ -69,6 +69,7 @@ arguments - which can alternatively be run by running the subcommand web.`
cmd.CmdDoctor,
cmd.CmdManager,
cmd.Cmdembedded,
cmd.CmdMigrateStorage,
}
// Now adjust these commands to add our global configuration options



+ 13
- 0
models/admin.go View File

@@ -9,6 +9,7 @@ import (
"os"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/timeutil"

"github.com/unknwon/com"
@@ -65,6 +66,18 @@ func RemoveAllWithNotice(title, path string) {
removeAllWithNotice(x, title, path)
}

// RemoveStorageWithNotice removes a file from the storage and
// creates a system notice when error occurs.
func RemoveStorageWithNotice(bucket storage.ObjectStorage, title, path string) {
if err := bucket.Delete(path); err != nil {
desc := fmt.Sprintf("%s [%s]: %v", title, path, err)
log.Warn(title+" [%s]: %v", path, err)
if err = createNotice(x, NoticeRepository, desc); err != nil {
log.Error("CreateRepositoryNotice: %v", err)
}
}
}

func removeAllWithNotice(e Engine, title, path string) {
if err := os.RemoveAll(path); err != nil {
desc := fmt.Sprintf("%s [%s]: %v", title, path, err)


+ 32
- 33
models/attachment.go View File

@@ -5,12 +5,13 @@
package models

import (
"bytes"
"fmt"
"io"
"os"
"path"

"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"

@@ -55,22 +56,16 @@ func (a *Attachment) APIFormat() *api.Attachment {
}
}

// AttachmentLocalPath returns where attachment is stored in local file
// system based on given UUID.
func AttachmentLocalPath(uuid string) string {
return path.Join(setting.AttachmentPath, uuid[0:1], uuid[1:2], uuid)
}

// LocalPath returns where attachment is stored in local file system.
func (a *Attachment) LocalPath() string {
return AttachmentLocalPath(a.UUID)
}

// DownloadURL returns the download url of the attached file
func (a *Attachment) DownloadURL() string {
return fmt.Sprintf("%sattachments/%s", setting.AppURL, a.UUID)
}

// RelativePath returns the relative path of the attachment
func (a *Attachment) RelativePath() string {
return path.Join(a.UUID[0:1], a.UUID[1:2], a.UUID)
}

// LinkedRepository returns the linked repo if any
func (a *Attachment) LinkedRepository() (*Repository, UnitType, error) {
if a.IssueID != 0 {
@@ -99,29 +94,11 @@ func (a *Attachment) LinkedRepository() (*Repository, UnitType, error) {
func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) {
attach.UUID = gouuid.NewV4().String()

localPath := attach.LocalPath()
if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
return nil, fmt.Errorf("MkdirAll: %v", err)
}

fw, err := os.Create(localPath)
size, err := storage.Attachments.Save(attach.RelativePath(), io.MultiReader(bytes.NewReader(buf), file))
if err != nil {
return nil, fmt.Errorf("Create: %v", err)
}
defer fw.Close()

if _, err = fw.Write(buf); err != nil {
return nil, fmt.Errorf("Write: %v", err)
} else if _, err = io.Copy(fw, file); err != nil {
return nil, fmt.Errorf("Copy: %v", err)
}

// Update file size
var fi os.FileInfo
if fi, err = fw.Stat(); err != nil {
return nil, fmt.Errorf("file size: %v", err)
}
attach.Size = fi.Size()
attach.Size = size

if _, err := x.Insert(attach); err != nil {
return nil, err
@@ -238,7 +215,7 @@ func DeleteAttachments(attachments []*Attachment, remove bool) (int, error) {

if remove {
for i, a := range attachments {
if err := os.Remove(a.LocalPath()); err != nil {
if storage.Attachments.Delete(a.RelativePath()); err != nil {
return i, err
}
}
@@ -290,3 +267,25 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
_, err := x.Where("release_id = ?", releaseID).Delete(&Attachment{})
return err
}

// IterateAttachment iterates attachments
func IterateAttachment(f func(attach *Attachment) error) error {
var start int
const batchSize = 100
for {
var attachments = make([]*Attachment, 0, batchSize)
if err := x.Limit(batchSize, start).Find(&attachments); err != nil {
return err
}
if len(attachments) == 0 {
return nil
}
start += len(attachments)

for _, attach := range attachments {
if err := f(attach); err != nil {
return err
}
}
}
}

+ 2
- 3
models/migrations/v112.go View File

@@ -5,9 +5,8 @@
package migrations

import (
"os"
"code.gitea.io/gitea/modules/storage"

"code.gitea.io/gitea/models"
"xorm.io/builder"
"xorm.io/xorm"
)
@@ -27,7 +26,7 @@ func removeAttachmentMissedRepo(x *xorm.Engine) error {
}

for i := 0; i < len(attachments); i++ {
os.RemoveAll(models.AttachmentLocalPath(attachments[i].UUID))
storage.Attachments.Delete(relativePath(attachments[i].UUID))
}

if len(attachments) < 50 {


+ 11
- 6
models/migrations/v96.go View File

@@ -5,16 +5,19 @@
package migrations

import (
"os"
"path"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"

"xorm.io/xorm"
)

func deleteOrphanedAttachments(x *xorm.Engine) error {
func relativePath(uuid string) string {
return path.Join(uuid[0:1], uuid[1:2], uuid)
}

func deleteOrphanedAttachments(x *xorm.Engine) error {
type Attachment struct {
ID int64 `xorm:"pk autoincr"`
UUID string `xorm:"uuid UNIQUE"`
@@ -47,12 +50,14 @@ func deleteOrphanedAttachments(x *xorm.Engine) error {
for _, attachment := range attachements {
ids = append(ids, attachment.ID)
}
if _, err := sess.In("id", ids).Delete(new(Attachment)); err != nil {
return err
if len(ids) > 0 {
if _, err := sess.In("id", ids).Delete(new(Attachment)); err != nil {
return err
}
}

for _, attachment := range attachements {
if err := os.RemoveAll(models.AttachmentLocalPath(attachment.UUID)); err != nil {
if err := storage.Attachments.Delete(relativePath(attachment.UUID)); err != nil {
return err
}
}


+ 5
- 4
models/repo.go View File

@@ -30,6 +30,7 @@ import (
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
@@ -1548,7 +1549,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
}
releaseAttachments := make([]string, 0, len(attachments))
for i := 0; i < len(attachments); i++ {
releaseAttachments = append(releaseAttachments, attachments[i].LocalPath())
releaseAttachments = append(releaseAttachments, attachments[i].UUID)
}

if err = deleteBeans(sess,
@@ -1627,7 +1628,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
}
attachmentPaths := make([]string, 0, len(attachments))
for j := range attachments {
attachmentPaths = append(attachmentPaths, attachments[j].LocalPath())
attachmentPaths = append(attachmentPaths, attachments[j].UUID)
}

if _, err = sess.In("issue_id", deleteCond).
@@ -1715,12 +1716,12 @@ func DeleteRepository(doer *User, uid, repoID int64) error {

// Remove issue attachment files.
for i := range attachmentPaths {
removeAllWithNotice(x, "Delete issue attachment", attachmentPaths[i])
RemoveStorageWithNotice(storage.Attachments, "Delete issue attachment", attachmentPaths[i])
}

// Remove release attachment files.
for i := range releaseAttachments {
removeAllWithNotice(x, "Delete release attachment", releaseAttachments[i])
RemoveStorageWithNotice(storage.Attachments, "Delete release attachment", releaseAttachments[i])
}

if len(repo.Avatar) > 0 {


+ 2
- 13
modules/migrations/gitea.go View File

@@ -13,7 +13,6 @@ import (
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"sync"
@@ -26,6 +25,7 @@ import (
"code.gitea.io/gitea/modules/repository"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"

@@ -275,18 +275,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
}
defer resp.Body.Close()

localPath := attach.LocalPath()
if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
return fmt.Errorf("MkdirAll: %v", err)
}

fw, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("Create: %v", err)
}
defer fw.Close()

_, err = io.Copy(fw, resp.Body)
_, err = storage.Attachments.Save(attach.RelativePath(), resp.Body)
return err
}()
if err != nil {


+ 62
- 0
modules/storage/local.go View File

@@ -0,0 +1,62 @@
package storage

import (
"io"
"os"
"path/filepath"
)

var (
_ ObjectStorage = &LocalStorage{}
)

// LocalStorage represents a local files storage
type LocalStorage struct {
dir string
}

// NewLocalStorage returns a local files
func NewLocalStorage(bucket string) (*LocalStorage, error) {
if err := os.MkdirAll(bucket, os.ModePerm); err != nil {
return nil, err
}

return &LocalStorage{
dir: bucket,
}, nil
}

// Open open a file
func (l *LocalStorage) Open(path string) (io.ReadCloser, error) {
f, err := os.Open(filepath.Join(l.dir, path))
if err != nil {
return nil, err
}
return f, nil
}

// Save save a file
func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
p := filepath.Join(l.dir, path)
if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
return 0, err
}

// always override
if err := os.RemoveAll(p); err != nil {
return 0, err
}

f, err := os.Create(p)
if err != nil {
return 0, err
}
defer f.Close()
return io.Copy(f, r)
}

// Delete delete a file
func (l *LocalStorage) Delete(path string) error {
p := filepath.Join(l.dir, path)
return os.Remove(p)
}

+ 59
- 0
modules/storage/minio.go View File

@@ -0,0 +1,59 @@
package storage

import (
"io"
"strings"

"github.com/minio/minio-go"
)

var (
_ ObjectStorage = &MinioStorage{}
)

// MinioStorage returns a minio bucket storage
type MinioStorage struct {
client *minio.Client
location string
bucket string
basePath string
}

// NewMinioStorage returns a minio storage
func NewMinioStorage(endpoint, accessKeyID, secretAccessKey, location, bucket, basePath string, useSSL bool) (*MinioStorage, error) {
minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
if err != nil {
return nil, err
}

return &MinioStorage{
location: location,
client: minioClient,
bucket: bucket,
basePath: basePath,
}, nil
}

func buildMinioPath(p string) string {
return strings.TrimPrefix(p, "/")
}

// Open open a file
func (m *MinioStorage) Open(path string) (io.ReadCloser, error) {
var opts = minio.GetObjectOptions{}
object, err := m.client.GetObject(m.bucket, buildMinioPath(path), opts)
if err != nil {
return nil, err
}
return object, nil
}

// Save save a file to minio
func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) {
return m.client.PutObject(m.bucket, buildMinioPath(path), r, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"})
}

// Delete delete a file
func (m *MinioStorage) Delete(path string) error {
return m.client.RemoveObject(m.bucket, buildMinioPath(path))
}

+ 42
- 0
modules/storage/storage.go View File

@@ -0,0 +1,42 @@
package storage

import (
"io"

"code.gitea.io/gitea/modules/setting"
)

// ObjectStorage represents an object storage to handle a bucket and files
type ObjectStorage interface {
Save(path string, r io.Reader) (int64, error)
Open(path string) (io.ReadCloser, error)
Delete(path string) error
}

// Copy copys a file from source ObjectStorage to dest ObjectStorage
func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, srcPath string) (int64, error) {
f, err := srcStorage.Open(srcPath)
if err != nil {
return 0, err
}
defer f.Close()

return dstStorage.Save(dstPath, f)
}

var (
// Attachments represents attachments storage
Attachments ObjectStorage
)

// Init init the stoarge
func Init() error {
var err error
Attachments, err = NewLocalStorage(setting.AttachmentPath)
if err != nil {
return err
}

return nil
}


+ 4
- 0
routers/init.go View File

@@ -28,6 +28,7 @@ import (
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/ssh"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/task"
"code.gitea.io/gitea/modules/webhook"
"code.gitea.io/gitea/services/mailer"
@@ -54,6 +55,9 @@ func checkRunMode() {
// NewServices init new services
func NewServices() {
setting.NewServices()
if err := storage.Init(); err != nil {
log.Fatal("storage init failed: %v", err)
}
mailer.NewContext()
_ = cache.NewContext()
notification.NewContext()


+ 2
- 2
routers/repo/attachment.go View File

@@ -7,13 +7,13 @@ package repo
import (
"fmt"
"net/http"
"os"
"strings"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/upload"
)

@@ -123,7 +123,7 @@ func GetAttachment(ctx *context.Context) {
}

//If we have matched and access to release or issue
fr, err := os.Open(attach.LocalPath())
fr, err := storage.Attachments.Open(attach.RelativePath())
if err != nil {
ctx.ServerError("Open", err)
return


+ 2
- 2
services/release/release.go View File

@@ -6,7 +6,6 @@ package release

import (
"fmt"
"os"
"strings"

"code.gitea.io/gitea/models"
@@ -14,6 +13,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/timeutil"
)

@@ -161,7 +161,7 @@ func DeleteReleaseByID(id int64, doer *models.User, delTag bool) error {

for i := range rel.Attachments {
attachment := rel.Attachments[i]
if err := os.RemoveAll(attachment.LocalPath()); err != nil {
if err := storage.Attachments.Delete(attachment.RelativePath()); err != nil {
log.Error("Delete attachment %s of release %s failed: %v", attachment.UUID, rel.ID, err)
}
}


+ 6
- 0
vendor/github.com/go-ini/ini/.gitignore View File

@@ -0,0 +1,6 @@
testdata/conf_out.ini
ini.sublime-project
ini.sublime-workspace
testdata/conf_reflect.ini
.idea
/.vscode

+ 191
- 0
vendor/github.com/go-ini/ini/LICENSE View File

@@ -0,0 +1,191 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/

TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION

1. Definitions.

"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.

"Licensor" shall mean the copyright owner or entity authorized by the copyright
owner that is granting the License.

"Legal Entity" shall mean the union of the acting entity and all other entities
that control, are controlled by, or are under common control with that entity.
For the purposes of this definition, "control" means (i) the power, direct or
indirect, to cause the direction or management of such entity, whether by
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.

"You" (or "Your") shall mean an individual or Legal Entity exercising
permissions granted by this License.

"Source" form shall mean the preferred form for making modifications, including
but not limited to software source code, documentation source, and configuration
files.

"Object" form shall mean any form resulting from mechanical transformation or
translation of a Source form, including but not limited to compiled object code,
generated documentation, and conversions to other media types.

"Work" shall mean the work of authorship, whether in Source or Object form, made
available under the License, as indicated by a copyright notice that is included
in or attached to the work (an example is provided in the Appendix below).

"Derivative Works" shall mean any work, whether in Source or Object form, that
is based on (or derived from) the Work and for which the editorial revisions,
annotations, elaborations, or other modifications represent, as a whole, an
original work of authorship. For the purposes of this License, Derivative Works
shall not include works that remain separable from, or merely link (or bind by
name) to the interfaces of, the Work and Derivative Works thereof.

"Contribution" shall mean any work of authorship, including the original version
of the Work and any modifications or additions to that Work or Derivative Works
thereof, that is intentionally submitted to Licensor for inclusion in the Work
by the copyright owner or by an individual or Legal Entity authorized to submit
on behalf of the copyright owner. For the purposes of this definition,
"submitted" means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems, and
issue tracking systems that are managed by, or on behalf of, the Licensor for
the purpose of discussing and improving the Work, but excluding communication
that is conspicuously marked or otherwise designated in writing by the copyright
owner as "Not a Contribution."

"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
of whom a Contribution has been received by Licensor and subsequently
incorporated within the Work.

2. Grant of Copyright License.

Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the Work and such
Derivative Works in Source or Object form.

3. Grant of Patent License.

Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to make, have
made, use, offer to sell, sell, import, and otherwise transfer the Work, where
such license applies only to those patent claims licensable by such Contributor
that are necessarily infringed by their Contribution(s) alone or by combination
of their Contribution(s) with the Work to which such Contribution(s) was
submitted. If You institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
Contribution incorporated within the Work constitutes direct or contributory
patent infringement, then any patent licenses granted to You under this License
for that Work shall terminate as of the date such litigation is filed.

4. Redistribution.

You may reproduce and distribute copies of the Work or Derivative Works thereof
in any medium, with or without modifications, and in Source or Object form,
provided that You meet the following conditions:

You must give any other recipients of the Work or Derivative Works a copy of
this License; and
You must cause any modified files to carry prominent notices stating that You
changed the files; and
You must retain, in the Source form of any Derivative Works that You distribute,
all copyright, patent, trademark, and attribution notices from the Source form
of the Work, excluding those notices that do not pertain to any part of the
Derivative Works; and
If the Work includes a "NOTICE" text file as part of its distribution, then any
Derivative Works that You distribute must include a readable copy of the
attribution notices contained within such NOTICE file, excluding those notices
that do not pertain to any part of the Derivative Works, in at least one of the
following places: within a NOTICE text file distributed as part of the
Derivative Works; within the Source form or documentation, if provided along
with the Derivative Works; or, within a display generated by the Derivative
Works, if and wherever such third-party notices normally appear. The contents of
the NOTICE file are for informational purposes only and do not modify the
License. You may add Your own attribution notices within Derivative Works that
You distribute, alongside or as an addendum to the NOTICE text from the Work,
provided that such additional attribution notices cannot be construed as
modifying the License.
You may add Your own copyright statement to Your modifications and may provide
additional or different license terms and conditions for use, reproduction, or
distribution of Your modifications, or for any such Derivative Works as a whole,
provided Your use, reproduction, and distribution of the Work otherwise complies
with the conditions stated in this License.

5. Submission of Contributions.

Unless You explicitly state otherwise, any Contribution intentionally submitted
for inclusion in the Work by You to the Licensor shall be under the terms and
conditions of this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify the terms of
any separate license agreement you may have executed with Licensor regarding
such Contributions.

6. Trademarks.

This License does not grant permission to use the trade names, trademarks,
service marks, or product names of the Licensor, except as required for
reasonable and customary use in describing the origin of the Work and
reproducing the content of the NOTICE file.

7. Disclaimer of Warranty.

Unless required by applicable law or agreed to in writing, Licensor provides the
Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
including, without limitation, any warranties or conditions of TITLE,
NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
solely responsible for determining the appropriateness of using or
redistributing the Work and assume any risks associated with Your exercise of
permissions under this License.

8. Limitation of Liability.

In no event and under no legal theory, whether in tort (including negligence),
contract, or otherwise, unless required by applicable law (such as deliberate
and grossly negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special, incidental,
or consequential damages of any character arising as a result of this License or
out of the use or inability to use the Work (including but not limited to
damages for loss of goodwill, work stoppage, computer failure or malfunction, or
any and all other commercial damages or losses), even if such Contributor has
been advised of the possibility of such damages.

9. Accepting Warranty or Additional Liability.

While redistributing the Work or Derivative Works thereof, You may choose to
offer, and charge a fee for, acceptance of support, warranty, indemnity, or
other liability obligations and/or rights consistent with this License. However,
in accepting such obligations, You may act only on Your own behalf and on Your
sole responsibility, not on behalf of any other Contributor, and only if You
agree to indemnify, defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason of your
accepting any such warranty or additional liability.

END OF TERMS AND CONDITIONS

APPENDIX: How to apply the Apache License to your work

To apply the Apache License to your work, attach the following boilerplate
notice, with the fields enclosed by brackets "[]" replaced with your own
identifying information. (Don't include the brackets!) The text should be
enclosed in the appropriate comment syntax for the file format. We also
recommend that a file or class name and description of purpose be included on
the same "printed page" as the copyright notice for easier identification within
third-party archives.

Copyright 2014 Unknwon

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

+ 15
- 0
vendor/github.com/go-ini/ini/Makefile View File

@@ -0,0 +1,15 @@
.PHONY: build test bench vet coverage

build: vet bench

test:
go test -v -cover -race

bench:
go test -v -cover -test.bench=. -test.benchmem

vet:
go vet

coverage:
go test -coverprofile=c.out && go tool cover -html=c.out && rm c.out

+ 43
- 0
vendor/github.com/go-ini/ini/README.md View File

@@ -0,0 +1,43 @@
# INI

[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/go-ini/ini/Go?logo=github&style=for-the-badge)](https://github.com/go-ini/ini/actions?query=workflow%3AGo)
[![codecov](https://img.shields.io/codecov/c/github/go-ini/ini/master?logo=codecov&style=for-the-badge)](https://codecov.io/gh/go-ini/ini)
[![GoDoc](https://img.shields.io/badge/GoDoc-Reference-blue?style=for-the-badge&logo=go)](https://pkg.go.dev/github.com/go-ini/ini?tab=doc)
[![Sourcegraph](https://img.shields.io/badge/view%20on-Sourcegraph-brightgreen.svg?style=for-the-badge&logo=sourcegraph)](https://sourcegraph.com/github.com/go-ini/ini)

![](https://avatars0.githubusercontent.com/u/10216035?v=3&s=200)

Package ini provides INI file read and write functionality in Go.

## Features

- Load from multiple data sources(file, `[]byte`, `io.Reader` and `io.ReadCloser`) with overwrites.
- Read with recursion values.
- Read with parent-child sections.
- Read with auto-increment key names.
- Read with multiple-line values.
- Read with tons of helper methods.
- Read and convert values to Go types.
- Read and **WRITE** comments of sections and keys.
- Manipulate sections, keys and comments with ease.
- Keep sections and keys in order as you parse and save.

## Installation

The minimum requirement of Go is **1.6**.

```sh
$ go get gopkg.in/ini.v1
```

Please add `-u` flag to update in the future.

## Getting Help

- [Getting Started](https://ini.unknwon.io/docs/intro/getting_started)
- [API Documentation](https://gowalker.org/gopkg.in/ini.v1)
- 中国大陆镜像:https://ini.unknwon.cn

## License

This project is under Apache v2 License. See the [LICENSE](LICENSE) file for the full license text.

+ 9
- 0
vendor/github.com/go-ini/ini/codecov.yml View File

@@ -0,0 +1,9 @@
coverage:
range: "60...95"
status:
project:
default:
threshold: 1%

comment:
layout: 'diff, files'

+ 76
- 0
vendor/github.com/go-ini/ini/data_source.go View File

@@ -0,0 +1,76 @@
// Copyright 2019 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ini

import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
)

var (
_ dataSource = (*sourceFile)(nil)
_ dataSource = (*sourceData)(nil)
_ dataSource = (*sourceReadCloser)(nil)
)

// dataSource is an interface that returns object which can be read and closed.
type dataSource interface {
ReadCloser() (io.ReadCloser, error)
}

// sourceFile represents an object that contains content on the local file system.
type sourceFile struct {
name string
}

func (s sourceFile) ReadCloser() (_ io.ReadCloser, err error) {
return os.Open(s.name)
}

// sourceData represents an object that contains content in memory.
type sourceData struct {
data []byte
}

func (s *sourceData) ReadCloser() (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewReader(s.data)), nil
}

// sourceReadCloser represents an input stream with Close method.
type sourceReadCloser struct {
reader io.ReadCloser
}

func (s *sourceReadCloser) ReadCloser() (io.ReadCloser, error) {
return s.reader, nil
}

func parseDataSource(source interface{}) (dataSource, error) {
switch s := source.(type) {
case string:
return sourceFile{s}, nil
case []byte:
return &sourceData{s}, nil
case io.ReadCloser:
return &sourceReadCloser{s}, nil
case io.Reader:
return &sourceReadCloser{ioutil.NopCloser(s)}, nil
default:
return nil, fmt.Errorf("error parsing data source: unknown type %q", s)
}
}

+ 25
- 0
vendor/github.com/go-ini/ini/deprecated.go View File

@@ -0,0 +1,25 @@
// Copyright 2019 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ini

const (
// Deprecated: Use "DefaultSection" instead.
DEFAULT_SECTION = DefaultSection
)

var (
// Deprecated: AllCapsUnderscore converts to format ALL_CAPS_UNDERSCORE.
AllCapsUnderscore = SnackCase
)

+ 34
- 0
vendor/github.com/go-ini/ini/error.go View File

@@ -0,0 +1,34 @@
// Copyright 2016 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ini

import (
"fmt"
)

// ErrDelimiterNotFound indicates the error type of no delimiter is found which there should be one.
type ErrDelimiterNotFound struct {
Line string
}

// IsErrDelimiterNotFound returns true if the given error is an instance of ErrDelimiterNotFound.
func IsErrDelimiterNotFound(err error) bool {
_, ok := err.(ErrDelimiterNotFound)
return ok
}

func (err ErrDelimiterNotFound) Error() string {
return fmt.Sprintf("key-value delimiter not found: %s", err.Line)
}

+ 509
- 0
vendor/github.com/go-ini/ini/file.go View File

@@ -0,0 +1,509 @@
// Copyright 2017 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ini

import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"sync"
)

// File represents a combination of one or more INI files in memory.
type File struct {
options LoadOptions
dataSources []dataSource

// Should make things safe, but sometimes doesn't matter.
BlockMode bool
lock sync.RWMutex

// To keep data in order.
sectionList []string
// To keep track of the index of a section with same name.
// This meta list is only used with non-unique section names are allowed.
sectionIndexes []int

// Actual data is stored here.
sections map[string][]*Section

NameMapper
ValueMapper
}

// newFile initializes File object with given data sources.
func newFile(dataSources []dataSource, opts LoadOptions) *File {
if len(opts.KeyValueDelimiters) == 0 {
opts.KeyValueDelimiters = "=:"
}
if len(opts.KeyValueDelimiterOnWrite) == 0 {
opts.KeyValueDelimiterOnWrite = "="
}

return &File{
BlockMode: true,
dataSources: dataSources,
sections: make(map[string][]*Section),
options: opts,
}
}

// Empty returns an empty file object.
func Empty(opts ...LoadOptions) *File {
var opt LoadOptions
if len(opts) > 0 {
opt = opts[0]
}

// Ignore error here, we are sure our data is good.
f, _ := LoadSources(opt, []byte(""))
return f
}

// NewSection creates a new section.
func (f *File) NewSection(name string) (*Section, error) {
if len(name) == 0 {
return nil, errors.New("empty section name")
}

if f.options.Insensitive && name != DefaultSection {
name = strings.ToLower(name)
}

if f.BlockMode {
f.lock.Lock()
defer f.lock.Unlock()
}

if !f.options.AllowNonUniqueSections && inSlice(name, f.sectionList) {
return f.sections[name][0], nil
}

f.sectionList = append(f.sectionList, name)

// NOTE: Append to indexes must happen before appending to sections,
// otherwise index will have off-by-one problem.
f.sectionIndexes = append(f.sectionIndexes, len(f.sections[name]))

sec := newSection(f, name)
f.sections[name] = append(f.sections[name], sec)

return sec, nil
}

// NewRawSection creates a new section with an unparseable body.
func (f *File) NewRawSection(name, body string) (*Section, error) {
section, err := f.NewSection(name)
if err != nil {
return nil, err
}

section.isRawSection = true
section.rawBody = body
return section, nil
}

// NewSections creates a list of sections.
func (f *File) NewSections(names ...string) (err error) {
for _, name := range names {
if _, err = f.NewSection(name); err != nil {
return err
}
}
return nil
}

// GetSection returns section by given name.
func (f *File) GetSection(name string) (*Section, error) {
secs, err := f.SectionsByName(name)
if err != nil {
return nil, err
}

return secs[0], err
}

// SectionsByName returns all sections with given name.
func (f *File) SectionsByName(name string) ([]*Section, error) {
if len(name) == 0 {
name = DefaultSection
}
if f.options.Insensitive {
name = strings.ToLower(name)
}

if f.BlockMode {
f.lock.RLock()
defer f.lock.RUnlock()
}

secs := f.sections[name]
if len(secs) == 0 {
return nil, fmt.Errorf("section %q does not exist", name)
}

return secs, nil
}

// Section assumes named section exists and returns a zero-value when not.
func (f *File) Section(name string) *Section {
sec, err := f.GetSection(name)
if err != nil {
// Note: It's OK here because the only possible error is empty section name,
// but if it's empty, this piece of code won't be executed.
sec, _ = f.NewSection(name)
return sec
}
return sec
}

// SectionWithIndex assumes named section exists and returns a new section when not.
func (f *File) SectionWithIndex(name string, index int) *Section {
secs, err := f.SectionsByName(name)
if err != nil || len(secs) <= index {
// NOTE: It's OK here because the only possible error is empty section name,
// but if it's empty, this piece of code won't be executed.
newSec, _ := f.NewSection(name)
return newSec
}

return secs[index]
}

// Sections returns a list of Section stored in the current instance.
func (f *File) Sections() []*Section {
if f.BlockMode {
f.lock.RLock()
defer f.lock.RUnlock()
}

sections := make([]*Section, len(f.sectionList))
for i, name := range f.sectionList {
sections[i] = f.sections[name][f.sectionIndexes[i]]
}
return sections
}

// ChildSections returns a list of child sections of given section name.
func (f *File) ChildSections(name string) []*Section {
return f.Section(name).ChildSections()
}

// SectionStrings returns list of section names.
func (f *File) SectionStrings() []string {
list := make([]string, len(f.sectionList))
copy(list, f.sectionList)
return list
}

// DeleteSection deletes a section or all sections with given name.
func (f *File) DeleteSection(name string) {
secs, err := f.SectionsByName(name)
if err != nil {
return
}

for i := 0; i < len(secs); i++ {
// For non-unique sections, it is always needed to remove the first one so
// in the next iteration, the subsequent section continue having index 0.
// Ignoring the error as index 0 never returns an error.
_ = f.DeleteSectionWithIndex(name, 0)
}
}

// DeleteSectionWithIndex deletes a section with given name and index.
func (f *File) DeleteSectionWithIndex(name string, index int) error {
if !f.options.AllowNonUniqueSections && index != 0 {
return fmt.Errorf("delete section with non-zero index is only allowed when non-unique sections is enabled")
}

if len(name) == 0 {
name = DefaultSection
}
if f.options.Insensitive {
name = strings.ToLower(name)
}

if f.BlockMode {
f.lock.Lock()
defer f.lock.Unlock()
}

// Count occurrences of the sections
occurrences := 0

sectionListCopy := make([]string, len(f.sectionList))
copy(sectionListCopy, f.sectionList)

for i, s := range sectionListCopy {
if s != name {
continue
}

if occurrences == index {
if len(f.sections[name]) <= 1 {
delete(f.sections, name) // The last one in the map
} else {
f.sections[name] = append(f.sections[name][:index], f.sections[name][index+1:]...)
}

// Fix section lists
f.sectionList = append(f.sectionList[:i], f.sectionList[i+1:]...)
f.sectionIndexes = append(f.sectionIndexes[:i], f.sectionIndexes[i+1:]...)

} else if occurrences > index {
// Fix the indices of all following sections with this name.
f.sectionIndexes[i-1]--
}

occurrences++
}

return nil
}

func (f *File) reload(s dataSource) error {
r, err := s.ReadCloser()
if err != nil {
return err
}
defer r.Close()

return f.parse(r)
}

// Reload reloads and parses all data sources.
func (f *File) Reload() (err error) {
for _, s := range f.dataSources {
if err = f.reload(s); err != nil {
// In loose mode, we create an empty default section for nonexistent files.
if os.IsNotExist(err) && f.options.Loose {
_ = f.parse(bytes.NewBuffer(nil))
continue
}
return err
}
}
return nil
}

// Append appends one or more data sources and reloads automatically.
func (f *File) Append(source interface{}, others ...interface{}) error {
ds, err := parseDataSource(source)
if err != nil {
return err
}
f.dataSources = append(f.dataSources, ds)
for _, s := range others {
ds, err = parseDataSource(s)
if err != nil {
return err
}
f.dataSources = append(f.dataSources, ds)
}
return f.Reload()
}

func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) {
equalSign := DefaultFormatLeft + f.options.KeyValueDelimiterOnWrite + DefaultFormatRight

if PrettyFormat || PrettyEqual {
equalSign = fmt.Sprintf(" %s ", f.options.KeyValueDelimiterOnWrite)
}

// Use buffer to make sure target is safe until finish encoding.
buf := bytes.NewBuffer(nil)
for i, sname := range f.sectionList {
sec := f.SectionWithIndex(sname, f.sectionIndexes[i])
if len(sec.Comment) > 0 {
// Support multiline comments
lines := strings.Split(sec.Comment, LineBreak)
for i := range lines {
if lines[i][0] != '#' && lines[i][0] != ';' {
lines[i] = "; " + lines[i]
} else {
lines[i] = lines[i][:1] + " " + strings.TrimSpace(lines[i][1:])
}

if _, err := buf.WriteString(lines[i] + LineBreak); err != nil {
return nil, err
}
}
}

if i > 0 || DefaultHeader {
if _, err := buf.WriteString("[" + sname + "]" + LineBreak); err != nil {
return nil, err
}
} else {
// Write nothing if default section is empty
if len(sec.keyList) == 0 {
continue
}
}

if sec.isRawSection {
if _, err := buf.WriteString(sec.rawBody); err != nil {
return nil, err
}

if PrettySection {
// Put a line between sections
if _, err := buf.WriteString(LineBreak); err != nil {
return nil, err
}
}
continue
}

// Count and generate alignment length and buffer spaces using the
// longest key. Keys may be modified if they contain certain characters so
// we need to take that into account in our calculation.
alignLength := 0
if PrettyFormat {
for _, kname := range sec.keyList {
keyLength := len(kname)
// First case will surround key by ` and second by """
if strings.Contains(kname, "\"") || strings.ContainsAny(kname, f.options.KeyValueDelimiters) {
keyLength += 2
} else if strings.Contains(kname, "`") {
keyLength += 6
}

if keyLength > alignLength {
alignLength = keyLength
}
}
}
alignSpaces := bytes.Repeat([]byte(" "), alignLength)

KeyList:
for _, kname := range sec.keyList {
key := sec.Key(kname)
if len(key.Comment) > 0 {
if len(indent) > 0 && sname != DefaultSection {
buf.WriteString(indent)
}

// Support multiline comments
lines := strings.Split(key.Comment, LineBreak)
for i := range lines {
if lines[i][0] != '#' && lines[i][0] != ';' {
lines[i] = "; " + strings.TrimSpace(lines[i])
} else {
lines[i] = lines[i][:1] + " " + strings.TrimSpace(lines[i][1:])
}

if _, err := buf.WriteString(lines[i] + LineBreak); err != nil {
return nil, err
}
}
}

if len(indent) > 0 && sname != DefaultSection {
buf.WriteString(indent)
}

switch {
case key.isAutoIncrement:
kname = "-"
case strings.Contains(kname, "\"") || strings.ContainsAny(kname, f.options.KeyValueDelimiters):
kname = "`" + kname + "`"
case strings.Contains(kname, "`"):
kname = `"""` + kname + `"""`
}

for _, val := range key.ValueWithShadows() {
if _, err := buf.WriteString(kname); err != nil {
return nil, err
}

if key.isBooleanType {
if kname != sec.keyList[len(sec.keyList)-1] {
buf.WriteString(LineBreak)
}
continue KeyList
}

// Write out alignment spaces before "=" sign
if PrettyFormat {
buf.Write(alignSpaces[:alignLength-len(kname)])
}

// In case key value contains "\n", "`", "\"", "#" or ";"
if strings.ContainsAny(val, "\n`") {
val = `"""` + val + `"""`
} else if !f.options.IgnoreInlineComment && strings.ContainsAny(val, "#;") {
val = "`" + val + "`"
}
if _, err := buf.WriteString(equalSign + val + LineBreak); err != nil {
return nil, err
}
}

for _, val := range key.nestedValues {
if _, err := buf.WriteString(indent + " " + val + LineBreak); err != nil {
return nil, err
}
}
}

if PrettySection {
// Put a line between sections
if _, err := buf.WriteString(LineBreak); err != nil {
return nil, err
}
}
}

return buf, nil
}

// WriteToIndent writes content into io.Writer with given indention.
// If PrettyFormat has been set to be true,
// it will align "=" sign with spaces under each section.
func (f *File) WriteToIndent(w io.Writer, indent string) (int64, error) {
buf, err := f.writeToBuffer(indent)
if err != nil {
return 0, err
}
return buf.WriteTo(w)
}

// WriteTo writes file content into io.Writer.
func (f *File) WriteTo(w io.Writer) (int64, error) {
return f.WriteToIndent(w, "")
}

// SaveToIndent writes content to file system with given value indention.
func (f *File) SaveToIndent(filename, indent string) error {
// Note: Because we are truncating with os.Create,
// so it's safer to save to a temporary file location and rename afte done.
buf, err := f.writeToBuffer(indent)
if err != nil {
return err
}

return ioutil.WriteFile(filename, buf.Bytes(), 0666)
}

// SaveTo writes content to file system.
func (f *File) SaveTo(filename string) error {
return f.SaveToIndent(filename, "")
}

+ 24
- 0
vendor/github.com/go-ini/ini/helper.go View File

@@ -0,0 +1,24 @@
// Copyright 2019 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ini

func inSlice(str string, s []string) bool {
for _, v := range s {
if str == v {
return true
}
}
return false
}

+ 168
- 0
vendor/github.com/go-ini/ini/ini.go View File

@@ -0,0 +1,168 @@
// +build go1.6

// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package ini provides INI file read and write functionality in Go.
package ini

import (
"os"
"regexp"
"runtime"
"strings"
)

const (
// DefaultSection is the name of default section. You can use this constant or the string literal.
// In most of cases, an empty string is all you need to access the section.
DefaultSection = "DEFAULT"

// Maximum allowed depth when recursively substituing variable names.
depthValues = 99
)

var (
// LineBreak is the delimiter to determine or compose a new line.
// This variable will be changed to "\r\n" automatically on Windows at package init time.
LineBreak = "\n"

// Variable regexp pattern: %(variable)s
varPattern = regexp.MustCompile(`%\(([^)]+)\)s`)

// DefaultHeader explicitly writes default section header.
DefaultHeader = false

// PrettySection indicates whether to put a line between sections.
PrettySection = true
// PrettyFormat indicates whether to align "=" sign with spaces to produce pretty output
// or reduce all possible spaces for compact format.
PrettyFormat = true
// PrettyEqual places spaces around "=" sign even when PrettyFormat is false.
PrettyEqual = false
// DefaultFormatLeft places custom spaces on the left when PrettyFormat and PrettyEqual are both disabled.
DefaultFormatLeft = ""
// DefaultFormatRight places custom spaces on the right when PrettyFormat and PrettyEqual are both disabled.
DefaultFormatRight = ""
)

var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test")

func init() {
if runtime.GOOS == "windows" && !inTest {
LineBreak = "\r\n"
}
}

// LoadOptions contains all customized options used for load data source(s).
type LoadOptions struct {
// Loose indicates whether the parser should ignore nonexistent files or return error.
Loose bool
// Insensitive indicates whether the parser forces all section and key names to lowercase.
Insensitive bool
// IgnoreContinuation indicates whether to ignore continuation lines while parsing.
IgnoreContinuation bool
// IgnoreInlineComment indicates whether to ignore comments at the end of value and treat it as part of value.
IgnoreInlineComment bool
// SkipUnrecognizableLines indicates whether to skip unrecognizable lines that do not conform to key/value pairs.
SkipUnrecognizableLines bool
// AllowBooleanKeys indicates whether to allow boolean type keys or treat as value is missing.
// This type of keys are mostly used in my.cnf.
AllowBooleanKeys bool
// AllowShadows indicates whether to keep track of keys with same name under same section.
AllowShadows bool
// AllowNestedValues indicates whether to allow AWS-like nested values.
// Docs: http://docs.aws.amazon.com/cli/latest/topic/config-vars.html#nested-values
AllowNestedValues bool
// AllowPythonMultilineValues indicates whether to allow Python-like multi-line values.
// Docs: https://docs.python.org/3/library/configparser.html#supported-ini-file-structure
// Relevant quote: Values can also span multiple lines, as long as they are indented deeper
// than the first line of the value.
AllowPythonMultilineValues bool
// SpaceBeforeInlineComment indicates whether to allow comment symbols (\# and \;) inside value.
// Docs: https://docs.python.org/2/library/configparser.html
// Quote: Comments may appear on their own in an otherwise empty line, or may be entered in lines holding values or section names.
// In the latter case, they need to be preceded by a whitespace character to be recognized as a comment.
SpaceBeforeInlineComment bool
// UnescapeValueDoubleQuotes indicates whether to unescape double quotes inside value to regular format
// when value is surrounded by double quotes, e.g. key="a \"value\"" => key=a "value"
UnescapeValueDoubleQuotes bool
// UnescapeValueCommentSymbols indicates to unescape comment symbols (\# and \;) inside value to regular format
// when value is NOT surrounded by any quotes.
// Note: UNSTABLE, behavior might change to only unescape inside double quotes but may noy necessary at all.
UnescapeValueCommentSymbols bool
// UnparseableSections stores a list of blocks that are allowed with raw content which do not otherwise
// conform to key/value pairs. Specify the names of those blocks here.
UnparseableSections []string
// KeyValueDelimiters is the sequence of delimiters that are used to separate key and value. By default, it is "=:".
KeyValueDelimiters string
// KeyValueDelimiters is the delimiter that are used to separate key and value output. By default, it is "=".
KeyValueDelimiterOnWrite string
// PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes).
PreserveSurroundedQuote bool
// DebugFunc is called to collect debug information (currently only useful to debug parsing Python-style multiline values).
DebugFunc DebugFunc
// ReaderBufferSize is the buffer size of the reader in bytes.
ReaderBufferSize int
// AllowNonUniqueSections indicates whether to allow sections with the same name multiple times.
AllowNonUniqueSections bool
}

// DebugFunc is the type of function called to log parse events.
type DebugFunc func(message string)

// LoadSources allows caller to apply customized options for loading from data source(s).
func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) {
sources := make([]dataSource, len(others)+1)
sources[0], err = parseDataSource(source)
if err != nil {
return nil, err
}
for i := range others {
sources[i+1], err = parseDataSource(others[i])
if err != nil {
return nil, err
}
}
f := newFile(sources, opts)
if err = f.Reload(); err != nil {
return nil, err
}
return f, nil
}

// Load loads and parses from INI data sources.
// Arguments can be mixed of file name with string type, or raw data in []byte.
// It will return error if list contains nonexistent files.
func Load(source interface{}, others ...interface{}) (*File, error) {
return LoadSources(LoadOptions{}, source, others...)
}

// LooseLoad has exactly same functionality as Load function
// except it ignores nonexistent files instead of returning error.
func LooseLoad(source interface{}, others ...interface{}) (*File, error) {
return LoadSources(LoadOptions{Loose: true}, source, others...)
}

// InsensitiveLoad has exactly same functionality as Load function
// except it forces all section and key names to be lowercased.
func InsensitiveLoad(source interface{}, others ...interface{}) (*File, error) {
return LoadSources(LoadOptions{Insensitive: true}, source, others...)
}

// ShadowLoad has exactly same functionality as Load function
// except it allows have shadow keys.
func ShadowLoad(source interface{}, others ...interface{}) (*File, error) {
return LoadSources(LoadOptions{AllowShadows: true}, source, others...)
}

+ 829
- 0
vendor/github.com/go-ini/ini/key.go View File

@@ -0,0 +1,829 @@
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ini

import (
"bytes"
"errors"
"fmt"
"strconv"
"strings"
"time"
)

// Key represents a key under a section.
type Key struct {
s *Section
Comment string
name string
value string
isAutoIncrement bool
isBooleanType bool

isShadow bool
shadows []*Key

nestedValues []string
}

// newKey simply return a key object with given values.
func newKey(s *Section, name, val string) *Key {
return &Key{
s: s,
name: name,
value: val,
}
}

func (k *Key) addShadow(val string) error {
if k.isShadow {
return errors.New("cannot add shadow to another shadow key")
} else if k.isAutoIncrement || k.isBooleanType {
return errors.New("cannot add shadow to auto-increment or boolean key")
}

// Deduplicate shadows based on their values.
if k.value == val {
return nil
}
for i := range k.shadows {
if k.shadows[i].value == val {
return nil
}
}

shadow := newKey(k.s, k.name, val)
shadow.isShadow = true
k.shadows = append(k.shadows, shadow)
return nil
}

// AddShadow adds a new shadow key to itself.
func (k *Key) AddShadow(val string) error {
if !k.s.f.options.AllowShadows {
return errors.New("shadow key is not allowed")
}
return k.addShadow(val)
}

func (k *Key) addNestedValue(val string) error {
if k.isAutoIncrement || k.isBooleanType {
return errors.New("cannot add nested value to auto-increment or boolean key")
}

k.nestedValues = append(k.nestedValues, val)
return nil
}

// AddNestedValue adds a nested value to the key.
func (k *Key) AddNestedValue(val string) error {
if !k.s.f.options.AllowNestedValues {
return errors.New("nested value is not allowed")
}
return k.addNestedValue(val)
}

// ValueMapper represents a mapping function for values, e.g. os.ExpandEnv
type ValueMapper func(string) string

// Name returns name of key.
func (k *Key) Name() string {
return k.name
}

// Value returns raw value of key for performance purpose.
func (k *Key) Value() string {
return k.value
}

// ValueWithShadows returns raw values of key and its shadows if any.
func (k *Key) ValueWithShadows() []string {
if len(k.shadows) == 0 {
return []string{k.value}
}
vals := make([]string, len(k.shadows)+1)
vals[0] = k.value
for i := range k.shadows {
vals[i+1] = k.shadows[i].value
}
return vals
}

// NestedValues returns nested values stored in the key.
// It is possible returned value is nil if no nested values stored in the key.
func (k *Key) NestedValues() []string {
return k.nestedValues
}

// transformValue takes a raw value and transforms to its final string.
func (k *Key) transformValue(val string) string {
if k.s.f.ValueMapper != nil {
val = k.s.f.ValueMapper(val)
}

// Fail-fast if no indicate char found for recursive value
if !strings.Contains(val, "%") {
return val
}
for i := 0; i < depthValues; i++ {
vr := varPattern.FindString(val)
if len(vr) == 0 {
break
}

// Take off leading '%(' and trailing ')s'.
noption := vr[2 : len(vr)-2]

// Search in the same section.
// If not found or found the key itself, then search again in default section.
nk, err := k.s.GetKey(noption)
if err != nil || k == nk {
nk, _ = k.s.f.Section("").GetKey(noption)
if nk == nil {
// Stop when no results found in the default section,
// and returns the value as-is.
break
}
}

// Substitute by new value and take off leading '%(' and trailing ')s'.
val = strings.Replace(val, vr, nk.value, -1)
}
return val
}

// String returns string representation of value.
func (k *Key) String() string {
return k.transformValue(k.value)
}

// Validate accepts a validate function which can
// return modifed result as key value.
func (k *Key) Validate(fn func(string) string) string {
return fn(k.String())
}

// parseBool returns the boolean value represented by the string.
//
// It accepts 1, t, T, TRUE, true, True, YES, yes, Yes, y, ON, on, On,
// 0, f, F, FALSE, false, False, NO, no, No, n, OFF, off, Off.
// Any other value returns an error.
func parseBool(str string) (value bool, err error) {
switch str {
case "1", "t", "T", "true", "TRUE", "True", "YES", "yes", "Yes", "y", "ON", "on", "On":
return true, nil
case "0", "f", "F", "false", "FALSE", "False", "NO", "no", "No", "n", "OFF", "off", "Off":
return false, nil
}
return false, fmt.Errorf("parsing \"%s\": invalid syntax", str)
}

// Bool returns bool type value.
func (k *Key) Bool() (bool, error) {
return parseBool(k.String())
}

// Float64 returns float64 type value.
func (k *Key) Float64() (float64, error) {