From 5968c63a11c94b0fdde0485af194bebb2ea1b8e7 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 14 May 2023 17:38:40 +0200 Subject: [PATCH] Add Go package registry (#24687) Fixes #7608 This PR adds a Go package registry usable with the Go proxy protocol. ![grafik](https://github.com/go-gitea/gitea/assets/1666336/328feb5c-3df2-4f9d-8eae-fe3126d14c37) --- custom/conf/app.example.ini | 2 + .../config-cheat-sheet.en-us.md | 1 + .../doc/usage/packages/debian.en-us.md | 4 +- .../doc/usage/packages/generic.en-us.md | 8 +- docs/content/doc/usage/packages/go.en-us.md | 77 ++++++ docs/content/doc/usage/packages/rpm.en-us.md | 4 +- models/packages/descriptor.go | 2 + models/packages/package.go | 6 + modules/packages/goproxy/metadata.go | 94 ++++++++ modules/packages/goproxy/metadata_test.go | 75 ++++++ modules/setting/packages.go | 2 + options/locale/locale_en-US.ini | 2 + public/img/svg/gitea-go.svg | 1 + routers/api/packages/api.go | 59 +++++ routers/api/packages/goproxy/goproxy.go | 226 ++++++++++++++++++ routers/api/v1/packages/package.go | 2 +- services/forms/package_form.go | 2 +- services/packages/packages.go | 2 + templates/package/content/go.tmpl | 14 ++ templates/package/view.tmpl | 1 + templates/swagger/v1_json.tmpl | 1 + .../integration/api_packages_goproxy_test.go | 166 +++++++++++++ web_src/svg/gitea-go.svg | 10 + 23 files changed, 751 insertions(+), 10 deletions(-) create mode 100644 docs/content/doc/usage/packages/go.en-us.md create mode 100644 modules/packages/goproxy/metadata.go create mode 100644 modules/packages/goproxy/metadata_test.go create mode 100644 public/img/svg/gitea-go.svg create mode 100644 routers/api/packages/goproxy/goproxy.go create mode 100644 templates/package/content/go.tmpl create mode 100644 tests/integration/api_packages_goproxy_test.go create mode 100644 web_src/svg/gitea-go.svg diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index fc0c50219..592257b3b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2463,6 +2463,8 @@ ROUTER = console ;LIMIT_SIZE_DEBIAN = -1 ;; Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_GENERIC = -1 +;; Maximum size of a Go upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_GO = -1 ;; Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_HELM = -1 ;; Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 82665d7d2..27e74f2a2 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -1223,6 +1223,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `LIMIT_SIZE_CONTAINER`: **-1**: Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_DEBIAN`: **-1**: Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_GENERIC`: **-1**: Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_GO`: **-1**: Maximum size of a Go upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_HELM`: **-1**: Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_MAVEN`: **-1**: Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_NPM`: **-1**: Maximum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/usage/packages/debian.en-us.md b/docs/content/doc/usage/packages/debian.en-us.md index 7506b5ce2..dc73da27c 100644 --- a/docs/content/doc/usage/packages/debian.en-us.md +++ b/docs/content/doc/usage/packages/debian.en-us.md @@ -83,7 +83,7 @@ curl --user your_username:your_password_or_token \ If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. You cannot publish a file with the same name twice to a package. You must delete the existing package version first. -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | @@ -115,7 +115,7 @@ curl --user your_username:your_token_or_password -X DELETE \ https://gitea.example.com/api/packages/testuser/debian/pools/bionic/main/test-package/1.0.0/amd64 ``` -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | diff --git a/docs/content/doc/usage/packages/generic.en-us.md b/docs/content/doc/usage/packages/generic.en-us.md index fbfe42d50..447eb692f 100644 --- a/docs/content/doc/usage/packages/generic.en-us.md +++ b/docs/content/doc/usage/packages/generic.en-us.md @@ -51,7 +51,7 @@ curl --user your_username:your_password_or_token \ If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | @@ -83,7 +83,7 @@ curl --user your_username:your_token_or_password \ https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0/file.bin ``` -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | @@ -111,7 +111,7 @@ curl --user your_username:your_token_or_password -X DELETE \ https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0 ``` -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | @@ -140,7 +140,7 @@ curl --user your_username:your_token_or_password -X DELETE \ https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0/file.bin ``` -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | diff --git a/docs/content/doc/usage/packages/go.en-us.md b/docs/content/doc/usage/packages/go.en-us.md new file mode 100644 index 000000000..92f5eb5e9 --- /dev/null +++ b/docs/content/doc/usage/packages/go.en-us.md @@ -0,0 +1,77 @@ +--- +date: "2023-05-10T00:00:00+00:00" +title: "Go Packages Repository" +slug: "go" +weight: 45 +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Go" + weight: 45 + identifier: "go" +--- + +# Go Packages Repository + +Publish Go packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Publish a package + +To publish a Go package perform a HTTP `PUT` operation with the package content in the request body. +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. +The package must follow the [documented structure](https://go.dev/ref/mod#zip-files). + +``` +PUT https://gitea.example.com/api/packages/{owner}/go/upload +``` + +| Parameter | Description | +| --------- | ----------- | +| `owner` | The owner of the package. | + +To authenticate to the package registry, you need to provide [custom HTTP headers or use HTTP Basic authentication]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}): + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/file.zip \ + https://gitea.example.com/api/packages/testuser/go/upload +``` + +If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. + +The server responds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `201 Created` | The package has been published. | +| `400 Bad Request` | The package is invalid. | +| `409 Conflict` | A package with the same name exist already. | + +## Install a package + +To install a Go package instruct Go to use the package registry as proxy: + +```shell +# use latest version +GOPROXY=https://gitea.example.com/api/packages/{owner}/go go install {package_name} +# or +GOPROXY=https://gitea.example.com/api/packages/{owner}/go go install {package_name}@latest +# use specific version +GOPROXY=https://gitea.example.com/api/packages/{owner}/go go install {package_name}@{package_version} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the package. | +| `package_name` | The package name. | +| `package_version` | The package version. | + +If the owner of the packages is private you need to [provide credentials](https://go.dev/ref/mod#private-module-proxy-auth). + +More information about the `GOPROXY` environment variable and how to protect against data leaks can be found in [the documentation](https://go.dev/ref/mod#private-modules). diff --git a/docs/content/doc/usage/packages/rpm.en-us.md b/docs/content/doc/usage/packages/rpm.en-us.md index 2f9bb539b..7b256046c 100644 --- a/docs/content/doc/usage/packages/rpm.en-us.md +++ b/docs/content/doc/usage/packages/rpm.en-us.md @@ -69,7 +69,7 @@ curl --user your_username:your_password_or_token \ If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. You cannot publish a file with the same name twice to a package. You must delete the existing package version first. -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | @@ -99,7 +99,7 @@ curl --user your_username:your_token_or_password -X DELETE \ https://gitea.example.com/api/packages/testuser/rpm/test-package/1.0.0/x86_64 ``` -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index a69f47711..8e0165086 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -155,6 +155,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &debian.Metadata{} case TypeGeneric: // generic packages have no metadata + case TypeGo: + // go packages have no metadata case TypeHelm: metadata = &helm.Metadata{} case TypeNuGet: diff --git a/models/packages/package.go b/models/packages/package.go index 17d4d79f3..2dfed7804 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -39,6 +39,7 @@ const ( TypeContainer Type = "container" TypeDebian Type = "debian" TypeGeneric Type = "generic" + TypeGo Type = "go" TypeHelm Type = "helm" TypeMaven Type = "maven" TypeNpm Type = "npm" @@ -61,6 +62,7 @@ var TypeList = []Type{ TypeContainer, TypeDebian, TypeGeneric, + TypeGo, TypeHelm, TypeMaven, TypeNpm, @@ -94,6 +96,8 @@ func (pt Type) Name() string { return "Debian" case TypeGeneric: return "Generic" + case TypeGo: + return "Go" case TypeHelm: return "Helm" case TypeMaven: @@ -139,6 +143,8 @@ func (pt Type) SVGName() string { return "gitea-debian" case TypeGeneric: return "octicon-package" + case TypeGo: + return "gitea-go" case TypeHelm: return "gitea-helm" case TypeMaven: diff --git a/modules/packages/goproxy/metadata.go b/modules/packages/goproxy/metadata.go new file mode 100644 index 000000000..40f7d2050 --- /dev/null +++ b/modules/packages/goproxy/metadata.go @@ -0,0 +1,94 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package goproxy + +import ( + "archive/zip" + "fmt" + "io" + "path" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +const ( + PropertyGoMod = "go.mod" + + maxGoModFileSize = 16 * 1024 * 1024 // https://go.dev/ref/mod#zip-path-size-constraints +) + +var ( + ErrInvalidStructure = util.NewInvalidArgumentErrorf("package has invalid structure") + ErrGoModFileTooLarge = util.NewInvalidArgumentErrorf("go.mod file is too large") +) + +type Package struct { + Name string + Version string + GoMod string +} + +// ParsePackage parses the Go package file +// https://go.dev/ref/mod#zip-files +func ParsePackage(r io.ReaderAt, size int64) (*Package, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + var p *Package + + for _, file := range archive.File { + nameAndVersion := path.Dir(file.Name) + + parts := strings.SplitN(nameAndVersion, "@", 2) + if len(parts) != 2 { + continue + } + + versionParts := strings.SplitN(parts[1], "/", 2) + + if p == nil { + p = &Package{ + Name: strings.TrimSuffix(nameAndVersion, "@"+parts[1]), + Version: versionParts[0], + } + } + + if len(versionParts) > 1 { + // files are expected in the "root" folder + continue + } + + if path.Base(file.Name) == "go.mod" { + if file.UncompressedSize64 > maxGoModFileSize { + return nil, ErrGoModFileTooLarge + } + + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + bytes, err := io.ReadAll(&io.LimitedReader{R: f, N: maxGoModFileSize}) + if err != nil { + return nil, err + } + + p.GoMod = string(bytes) + + return p, nil + } + } + + if p == nil { + return nil, ErrInvalidStructure + } + + p.GoMod = fmt.Sprintf("module %s", p.Name) + + return p, nil +} diff --git a/modules/packages/goproxy/metadata_test.go b/modules/packages/goproxy/metadata_test.go new file mode 100644 index 000000000..4e7f394f8 --- /dev/null +++ b/modules/packages/goproxy/metadata_test.go @@ -0,0 +1,75 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package goproxy + +import ( + "archive/zip" + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + packageName = "gitea.com/go-gitea/gitea" + packageVersion = "v0.0.1" +) + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) *bytes.Reader { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, content := range files { + w, _ := zw.Create(name) + w.Write(content) + } + zw.Close() + return bytes.NewReader(buf.Bytes()) + } + + t.Run("EmptyPackage", func(t *testing.T) { + data := createArchive(nil) + + p, err := ParsePackage(data, int64(data.Len())) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidStructure) + }) + + t.Run("InvalidNameOrVersionStructure", func(t *testing.T) { + data := createArchive(map[string][]byte{ + packageName + "/" + packageVersion + "/go.mod": {}, + }) + + p, err := ParsePackage(data, int64(data.Len())) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidStructure) + }) + + t.Run("GoModFileInWrongDirectory", func(t *testing.T) { + data := createArchive(map[string][]byte{ + packageName + "@" + packageVersion + "/subdir/go.mod": {}, + }) + + p, err := ParsePackage(data, int64(data.Len())) + assert.NotNil(t, p) + assert.NoError(t, err) + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, "module gitea.com/go-gitea/gitea", p.GoMod) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive(map[string][]byte{ + packageName + "@" + packageVersion + "/subdir/go.mod": []byte("invalid"), + packageName + "@" + packageVersion + "/go.mod": []byte("valid"), + }) + + p, err := ParsePackage(data, int64(data.Len())) + assert.NotNil(t, p) + assert.NoError(t, err) + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, "valid", p.GoMod) + }) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 3719e2f64..a9b91adf1 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -33,6 +33,7 @@ var ( LimitSizeContainer int64 LimitSizeDebian int64 LimitSizeGeneric int64 + LimitSizeGo int64 LimitSizeHelm int64 LimitSizeMaven int64 LimitSizeNpm int64 @@ -79,6 +80,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) { Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER") Packages.LimitSizeDebian = mustBytes(sec, "LIMIT_SIZE_DEBIAN") Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC") + Packages.LimitSizeGo = mustBytes(sec, "LIMIT_SIZE_GO") Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM") Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN") Packages.LimitSizeNpm = mustBytes(sec, "LIMIT_SIZE_NPM") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 18b8bffe4..0385c28c4 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3263,6 +3263,8 @@ debian.repository.components = Components debian.repository.architectures = Architectures generic.download = Download package from the command line: generic.documentation = For more information on the generic registry, see the documentation. +go.install = Install the package from the command line: +go.documentation = For more information on the Go registry, see the documentation. helm.registry = Setup this registry from the command line: helm.install = To install the package, run the following command: helm.documentation = For more information on the Helm registry, see the documentation. diff --git a/public/img/svg/gitea-go.svg b/public/img/svg/gitea-go.svg new file mode 100644 index 000000000..a432bdbf2 --- /dev/null +++ b/public/img/svg/gitea-go.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 355387332..aaceb8a92 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/routers/api/packages/container" "code.gitea.io/gitea/routers/api/packages/debian" "code.gitea.io/gitea/routers/api/packages/generic" + "code.gitea.io/gitea/routers/api/packages/goproxy" "code.gitea.io/gitea/routers/api/packages/helm" "code.gitea.io/gitea/routers/api/packages/maven" "code.gitea.io/gitea/routers/api/packages/npm" @@ -312,6 +313,64 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { }, reqPackageAccess(perm.AccessModeWrite)) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/go", func() { + r.Put("/upload", reqPackageAccess(perm.AccessModeWrite), goproxy.UploadPackage) + r.Get("/sumdb/sum.golang.org/supported", func(ctx *context.Context) { + ctx.Status(http.StatusNotFound) + }) + + // Manual mapping of routes because the package name contains slashes which chi does not support + // https://go.dev/ref/mod#goproxy-protocol + r.Get("/*", func(ctx *context.Context) { + path := ctx.Params("*") + + if strings.HasSuffix(path, "/@latest") { + ctx.SetParams("name", path[:len(path)-len("/@latest")]) + ctx.SetParams("version", "latest") + + goproxy.PackageVersionMetadata(ctx) + return + } + + parts := strings.SplitN(path, "/@v/", 2) + if len(parts) != 2 { + ctx.Status(http.StatusNotFound) + return + } + + ctx.SetParams("name", parts[0]) + + // /@v/list + if parts[1] == "list" { + goproxy.EnumeratePackageVersions(ctx) + return + } + + // /@v/.zip + if strings.HasSuffix(parts[1], ".zip") { + ctx.SetParams("version", parts[1][:len(parts[1])-len(".zip")]) + + goproxy.DownloadPackageFile(ctx) + return + } + // /@v/.info + if strings.HasSuffix(parts[1], ".info") { + ctx.SetParams("version", parts[1][:len(parts[1])-len(".info")]) + + goproxy.PackageVersionMetadata(ctx) + return + } + // /@v/.mod + if strings.HasSuffix(parts[1], ".mod") { + ctx.SetParams("version", parts[1][:len(parts[1])-len(".mod")]) + + goproxy.PackageVersionGoModContent(ctx) + return + } + + ctx.Status(http.StatusNotFound) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/generic", func() { r.Group("/{packagename}/{packageversion}", func() { r.Delete("", reqPackageAccess(perm.AccessModeWrite), generic.DeletePackage) diff --git a/routers/api/packages/goproxy/goproxy.go b/routers/api/packages/goproxy/goproxy.go new file mode 100644 index 000000000..d0bc9c1e9 --- /dev/null +++ b/routers/api/packages/goproxy/goproxy.go @@ -0,0 +1,226 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package goproxy + +import ( + "errors" + "fmt" + "io" + "net/http" + "sort" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + goproxy_module "code.gitea.io/gitea/modules/packages/goproxy" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +func EnumeratePackageVersions(ctx *context.Context) { + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeGo, ctx.Params("name")) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + sort.Slice(pvs, func(i, j int) bool { + return pvs[i].CreatedUnix < pvs[j].CreatedUnix + }) + + ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") + + for _, pv := range pvs { + fmt.Fprintln(ctx.Resp, pv.Version) + } +} + +func PackageVersionMetadata(ctx *context.Context) { + pv, err := resolvePackage(ctx, ctx.Package.Owner.ID, ctx.Params("name"), ctx.Params("version")) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + ctx.JSON(http.StatusOK, struct { + Version string `json:"Version"` + Time time.Time `json:"Time"` + }{ + Version: pv.Version, + Time: pv.CreatedUnix.AsLocalTime(), + }) +} + +func PackageVersionGoModContent(ctx *context.Context) { + pv, err := resolvePackage(ctx, ctx.Package.Owner.ID, ctx.Params("name"), ctx.Params("version")) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, goproxy_module.PropertyGoMod) + if err != nil || len(pps) != 1 { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.PlainText(http.StatusOK, pps[0].Value) +} + +func DownloadPackageFile(ctx *context.Context) { + pv, err := resolvePackage(ctx, ctx.Package.Owner.ID, ctx.Params("name"), ctx.Params("version")) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil || len(pfs) != 1 { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + s, _, err := packages_service.GetPackageFileStream(ctx, pfs[0]) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + defer s.Close() + + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pfs[0].Name, + LastModified: pfs[0].CreatedUnix.AsLocalTime(), + }) +} + +func resolvePackage(ctx *context.Context, ownerID int64, name, version string) (*packages_model.PackageVersion, error) { + var pv *packages_model.PackageVersion + + if version == "latest" { + pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ownerID, + Type: packages_model.TypeGo, + Name: packages_model.SearchValue{ + Value: name, + ExactMatch: true, + }, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + }) + if err != nil { + return nil, err + } + + if len(pvs) != 1 { + return nil, packages_model.ErrPackageNotExist + } + + pv = pvs[0] + } else { + var err error + pv, err = packages_model.GetVersionByNameAndVersion(ctx, ownerID, packages_model.TypeGo, name, version) + if err != nil { + return nil, err + } + } + + return pv, nil +} + +func UploadPackage(ctx *context.Context) { + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if close { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + pck, err := goproxy_module.ParsePackage(buf, buf.Size()) + if err != nil { + if errors.Is(err, util.ErrInvalidArgument) { + apiError(ctx, http.StatusBadRequest, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeGo, + Name: pck.Name, + Version: pck.Version, + }, + Creator: ctx.Doer, + VersionProperties: map[string]string{ + goproxy_module.PropertyGoMod: pck.GoMod, + }, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: fmt.Sprintf("%v.zip", pck.Version), + }, + Creator: ctx.Doer, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion: + apiError(ctx, http.StatusConflict, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + ctx.Status(http.StatusCreated) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index d7277247f..0c9a13428 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [alpine, cargo, chef, composer, conan, conda, container, debian, generic, helm, maven, npm, nuget, pub, pypi, rpm, rubygems, swift, vagrant] + // enum: [alpine, cargo, chef, composer, conan, conda, container, debian, generic, go, helm, maven, npm, nuget, pub, pypi, rpm, rubygems, swift, vagrant] // - name: q // in: query // description: name filter diff --git a/services/forms/package_form.go b/services/forms/package_form.go index 96209ec84..dfec98fff 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,debian,generic,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"` + Type string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/packages.go b/services/packages/packages.go index bf64890f4..9d5ce04a0 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -369,6 +369,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p typeSpecificSize = setting.Packages.LimitSizeDebian case packages_model.TypeGeneric: typeSpecificSize = setting.Packages.LimitSizeGeneric + case packages_model.TypeGo: + typeSpecificSize = setting.Packages.LimitSizeGo case packages_model.TypeHelm: typeSpecificSize = setting.Packages.LimitSizeHelm case packages_model.TypeMaven: diff --git a/templates/package/content/go.tmpl b/templates/package/content/go.tmpl new file mode 100644 index 000000000..2343d945b --- /dev/null +++ b/templates/package/content/go.tmpl @@ -0,0 +1,14 @@ +{{if eq .PackageDescriptor.Package.Type "go"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
GOPROXY= go install {{$.PackageDescriptor.Package.Name}}@{{$.PackageDescriptor.Version.Version}}
+
+
+ +
+
+
+{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 3c42c3adf..5285a0838 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -28,6 +28,7 @@ {{template "package/content/container" .}} {{template "package/content/debian" .}} {{template "package/content/generic" .}} + {{template "package/content/go" .}} {{template "package/content/helm" .}} {{template "package/content/maven" .}} {{template "package/content/npm" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 01055cb00..3859eb556 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2418,6 +2418,7 @@ "container", "debian", "generic", + "go", "helm", "maven", "npm", diff --git a/tests/integration/api_packages_goproxy_test.go b/tests/integration/api_packages_goproxy_test.go new file mode 100644 index 000000000..08c1ca54f --- /dev/null +++ b/tests/integration/api_packages_goproxy_test.go @@ -0,0 +1,166 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/zip" + "bytes" + "fmt" + "net/http" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestPackageGo(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + packageName := "gitea.com/go-gitea/gitea" + packageVersion := "v0.0.1" + packageVersion2 := "v0.0.2" + goModContent := `module "gitea.com/go-gitea/gitea"` + + createArchive := func(files map[string][]byte) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, content := range files { + w, _ := zw.Create(name) + w.Write(content) + } + zw.Close() + return buf.Bytes() + } + + url := fmt.Sprintf("/api/packages/%s/go", user.Name) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := createArchive(nil) + + req := NewRequestWithBody(t, "PUT", url+"/upload", bytes.NewReader(content)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", url+"/upload", bytes.NewReader(content)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + + content = createArchive(map[string][]byte{ + packageName + "@" + packageVersion + "/go.mod": []byte(goModContent), + }) + + req = NewRequestWithBody(t, "PUT", url+"/upload", bytes.NewReader(content)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeGo) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.Nil(t, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, packageVersion+".zip", pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) + + req = NewRequestWithBody(t, "PUT", url+"/upload", bytes.NewReader(content)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusConflict) + + time.Sleep(time.Second) + + content = createArchive(map[string][]byte{ + packageName + "@" + packageVersion2 + "/go.mod": []byte(goModContent), + }) + + req = NewRequestWithBody(t, "PUT", url+"/upload", bytes.NewReader(content)) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + }) + + t.Run("List", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/@v/list", url, packageName)) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, packageVersion+"\n"+packageVersion2+"\n", resp.Body.String()) + }) + + t.Run("Info", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/@v/%s.info", url, packageName, packageVersion)) + resp := MakeRequest(t, req, http.StatusOK) + + type Info struct { + Version string `json:"Version"` + Time time.Time `json:"Time"` + } + + info := &Info{} + DecodeJSON(t, resp, &info) + + assert.Equal(t, packageVersion, info.Version) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/@v/latest.info", url, packageName)) + resp = MakeRequest(t, req, http.StatusOK) + + info = &Info{} + DecodeJSON(t, resp, &info) + + assert.Equal(t, packageVersion2, info.Version) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/@latest", url, packageName)) + resp = MakeRequest(t, req, http.StatusOK) + + info = &Info{} + DecodeJSON(t, resp, &info) + + assert.Equal(t, packageVersion2, info.Version) + }) + + t.Run("GoMod", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/@v/%s.mod", url, packageName, packageVersion)) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, goModContent, resp.Body.String()) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/@v/latest.mod", url, packageName)) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, goModContent, resp.Body.String()) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/@v/%s.zip", url, packageName, packageVersion)) + MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/@v/latest.zip", url, packageName)) + MakeRequest(t, req, http.StatusOK) + }) +} diff --git a/web_src/svg/gitea-go.svg b/web_src/svg/gitea-go.svg new file mode 100644 index 000000000..84e201ed1 --- /dev/null +++ b/web_src/svg/gitea-go.svg @@ -0,0 +1,10 @@ + + + + + + + + + +