gitea/services/packages/packages.go
Danila Fominykh 60522fc96f
packages: Calculate package size quota using package creator ID instead of owner ID (#28007)
Changed behavior to calculate package quota limit using package `creator
ID` instead of `owner ID`.

Currently, users are allowed to create an unlimited number of
organizations, each of which has its own package limit quota, resulting
in the ability for users to have unlimited package space in different
organization scopes. This fix will calculate package quota based on
`package version creator ID` instead of `package version owner ID`
(which might be organization), so that users are not allowed to take
more space than configured package settings.

Also, there is a side case in which users can publish packages to a
specific package version, initially published by different user, taking
that user package size quota. Version in fix should be better because
the total amount of space is limited to the quota for users sharing the
same organization scope.
2023-11-14 00:38:50 +01:00

658 lines
21 KiB
Go

// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package packages
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"net/url"
"strings"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
notify_service "code.gitea.io/gitea/services/notify"
)
var (
ErrQuotaTypeSize = errors.New("maximum allowed package type size exceeded")
ErrQuotaTotalSize = errors.New("maximum allowed package storage quota exceeded")
ErrQuotaTotalCount = errors.New("maximum allowed package count exceeded")
)
// PackageInfo describes a package
type PackageInfo struct {
Owner *user_model.User
PackageType packages_model.Type
Name string
Version string
}
// PackageCreationInfo describes a package to create
type PackageCreationInfo struct {
PackageInfo
SemverCompatible bool
Creator *user_model.User
Metadata any
PackageProperties map[string]string
VersionProperties map[string]string
}
// PackageFileInfo describes a package file
type PackageFileInfo struct {
Filename string
CompositeKey string
}
// PackageFileCreationInfo describes a package file to create
type PackageFileCreationInfo struct {
PackageFileInfo
Creator *user_model.User
Data packages_module.HashedSizeReader
IsLead bool
Properties map[string]string
OverwriteExisting bool
}
// CreatePackageAndAddFile creates a package with a file. If the same package exists already, ErrDuplicatePackageVersion is returned
func CreatePackageAndAddFile(ctx context.Context, pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
return createPackageAndAddFile(ctx, pvci, pfci, false)
}
// CreatePackageOrAddFileToExisting creates a package with a file or adds the file if the package exists already
func CreatePackageOrAddFileToExisting(ctx context.Context, pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
return createPackageAndAddFile(ctx, pvci, pfci, true)
}
func createPackageAndAddFile(ctx context.Context, pvci *PackageCreationInfo, pfci *PackageFileCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
dbCtx, committer, err := db.TxContext(ctx)
if err != nil {
return nil, nil, err
}
defer committer.Close()
pv, created, err := createPackageAndVersion(dbCtx, pvci, allowDuplicate)
if err != nil {
return nil, nil, err
}
pf, pb, blobCreated, err := addFileToPackageVersion(dbCtx, pv, &pvci.PackageInfo, pfci)
removeBlob := false
defer func() {
if blobCreated && removeBlob {
contentStore := packages_module.NewContentStore()
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob from content store: %v", err)
}
}
}()
if err != nil {
removeBlob = true
return nil, nil, err
}
if err := committer.Commit(); err != nil {
removeBlob = true
return nil, nil, err
}
if created {
pd, err := packages_model.GetPackageDescriptor(ctx, pv)
if err != nil {
return nil, nil, err
}
notify_service.PackageCreate(ctx, pvci.Creator, pd)
}
return pv, pf, nil
}
func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, bool, error) {
log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %+v, %v", pvci.Creator.ID, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, pvci.PackageProperties, pvci.VersionProperties, allowDuplicate)
packageCreated := true
p := &packages_model.Package{
OwnerID: pvci.Owner.ID,
Type: pvci.PackageType,
Name: pvci.Name,
LowerName: strings.ToLower(pvci.Name),
SemverCompatible: pvci.SemverCompatible,
}
var err error
if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
if err == packages_model.ErrDuplicatePackage {
packageCreated = false
} else {
log.Error("Error inserting package: %v", err)
return nil, false, err
}
}
if packageCreated {
for name, value := range pvci.PackageProperties {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, name, value); err != nil {
log.Error("Error setting package property: %v", err)
return nil, false, err
}
}
}
metadataJSON, err := json.Marshal(pvci.Metadata)
if err != nil {
return nil, false, err
}
versionCreated := true
pv := &packages_model.PackageVersion{
PackageID: p.ID,
CreatorID: pvci.Creator.ID,
Version: pvci.Version,
LowerVersion: strings.ToLower(pvci.Version),
MetadataJSON: string(metadataJSON),
}
if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
versionCreated = false
}
if err != packages_model.ErrDuplicatePackageVersion || !allowDuplicate {
log.Error("Error inserting package: %v", err)
return nil, false, err
}
}
if versionCreated {
if err := CheckCountQuotaExceeded(ctx, pvci.Creator, pvci.Owner); err != nil {
return nil, false, err
}
for name, value := range pvci.VersionProperties {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil {
log.Error("Error setting package version property: %v", err)
return nil, false, err
}
}
}
return pv, versionCreated, nil
}
// AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned
func AddFileToExistingPackage(ctx context.Context, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) {
return addFileToPackageWrapper(ctx, func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
if err != nil {
return nil, nil, false, err
}
return addFileToPackageVersion(ctx, pv, pvi, pfci)
})
}
// AddFileToPackageVersionInternal adds a file to the package
// This method skips quota checks and should only be used for system-managed packages.
func AddFileToPackageVersionInternal(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) {
return addFileToPackageWrapper(ctx, func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
return addFileToPackageVersionUnchecked(ctx, pv, pfci)
})
}
func addFileToPackageWrapper(ctx context.Context, fn func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error)) (*packages_model.PackageFile, error) {
ctx, committer, err := db.TxContext(ctx)
if err != nil {
return nil, err
}
defer committer.Close()
pf, pb, blobCreated, err := fn(ctx)
removeBlob := false
defer func() {
if removeBlob {
contentStore := packages_module.NewContentStore()
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob from content store: %v", err)
}
}
}()
if err != nil {
removeBlob = blobCreated
return nil, err
}
if err := committer.Commit(); err != nil {
removeBlob = blobCreated
return nil, err
}
return pf, nil
}
// NewPackageBlob creates a package blob instance
func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.PackageBlob {
hashMD5, hashSHA1, hashSHA256, hashSHA512 := hsr.Sums()
return &packages_model.PackageBlob{
Size: hsr.Size(),
HashMD5: hex.EncodeToString(hashMD5),
HashSHA1: hex.EncodeToString(hashSHA1),
HashSHA256: hex.EncodeToString(hashSHA256),
HashSHA512: hex.EncodeToString(hashSHA512),
}
}
func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
if err := CheckSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil {
return nil, nil, false, err
}
return addFileToPackageVersionUnchecked(ctx, pv, pfci)
}
func addFileToPackageVersionUnchecked(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
if err != nil {
log.Error("Error inserting package blob: %v", err)
return nil, nil, false, err
}
if !exists {
contentStore := packages_module.NewContentStore()
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), pfci.Data, pfci.Data.Size()); err != nil {
log.Error("Error saving package blob in content store: %v", err)
return nil, nil, false, err
}
}
if pfci.OverwriteExisting {
pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfci.Filename, pfci.CompositeKey)
if err != nil && err != packages_model.ErrPackageFileNotExist {
return nil, pb, !exists, err
}
if pf != nil {
// Short circuit if blob is the same
if pf.BlobID == pb.ID {
return pf, pb, !exists, nil
}
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
return nil, pb, !exists, err
}
if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
return nil, pb, !exists, err
}
}
}
pf := &packages_model.PackageFile{
VersionID: pv.ID,
BlobID: pb.ID,
Name: pfci.Filename,
LowerName: strings.ToLower(pfci.Filename),
CompositeKey: pfci.CompositeKey,
IsLead: pfci.IsLead,
}
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
if err != packages_model.ErrDuplicatePackageFile {
log.Error("Error inserting package file: %v", err)
}
return nil, pb, !exists, err
}
for name, value := range pfci.Properties {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil {
log.Error("Error setting package file property: %v", err)
return pf, pb, !exists, err
}
}
return pf, pb, !exists, nil
}
// CheckCountQuotaExceeded checks if the owner has more than the allowed packages
// The check is skipped if the doer is an admin.
func CheckCountQuotaExceeded(ctx context.Context, doer, owner *user_model.User) error {
if doer.IsAdmin {
return nil
}
if setting.Packages.LimitTotalOwnerCount > -1 {
totalCount, err := packages_model.CountVersions(ctx, &packages_model.PackageSearchOptions{
OwnerID: owner.ID,
IsInternal: util.OptionalBoolFalse,
})
if err != nil {
log.Error("CountVersions failed: %v", err)
return err
}
if totalCount > setting.Packages.LimitTotalOwnerCount {
return ErrQuotaTotalCount
}
}
return nil
}
// CheckSizeQuotaExceeded checks if the upload size is bigger than the allowed size
// The check is skipped if the doer is an admin.
func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, packageType packages_model.Type, uploadSize int64) error {
if doer.IsAdmin {
return nil
}
var typeSpecificSize int64
switch packageType {
case packages_model.TypeAlpine:
typeSpecificSize = setting.Packages.LimitSizeAlpine
case packages_model.TypeCargo:
typeSpecificSize = setting.Packages.LimitSizeCargo
case packages_model.TypeChef:
typeSpecificSize = setting.Packages.LimitSizeChef
case packages_model.TypeComposer:
typeSpecificSize = setting.Packages.LimitSizeComposer
case packages_model.TypeConan:
typeSpecificSize = setting.Packages.LimitSizeConan
case packages_model.TypeConda:
typeSpecificSize = setting.Packages.LimitSizeConda
case packages_model.TypeContainer:
typeSpecificSize = setting.Packages.LimitSizeContainer
case packages_model.TypeCran:
typeSpecificSize = setting.Packages.LimitSizeCran
case packages_model.TypeDebian:
typeSpecificSize = setting.Packages.LimitSizeDebian
case packages_model.TypeGeneric:
typeSpecificSize = setting.Packages.LimitSizeGeneric
case packages_model.TypeGo:
typeSpecificSize = setting.Packages.LimitSizeGo
case packages_model.TypeHelm:
typeSpecificSize = setting.Packages.LimitSizeHelm
case packages_model.TypeMaven:
typeSpecificSize = setting.Packages.LimitSizeMaven
case packages_model.TypeNpm:
typeSpecificSize = setting.Packages.LimitSizeNpm
case packages_model.TypeNuGet:
typeSpecificSize = setting.Packages.LimitSizeNuGet
case packages_model.TypePub:
typeSpecificSize = setting.Packages.LimitSizePub
case packages_model.TypePyPI:
typeSpecificSize = setting.Packages.LimitSizePyPI
case packages_model.TypeRpm:
typeSpecificSize = setting.Packages.LimitSizeRpm
case packages_model.TypeRubyGems:
typeSpecificSize = setting.Packages.LimitSizeRubyGems
case packages_model.TypeSwift:
typeSpecificSize = setting.Packages.LimitSizeSwift
case packages_model.TypeVagrant:
typeSpecificSize = setting.Packages.LimitSizeVagrant
}
if typeSpecificSize > -1 && typeSpecificSize < uploadSize {
return ErrQuotaTypeSize
}
if setting.Packages.LimitTotalOwnerSize > -1 {
totalSize, err := packages_model.CalculateCreatorPackageQuota(ctx, doer.ID)
if err != nil {
log.Error("CalculateFileSize failed: %v", err)
return err
}
if totalSize+uploadSize > setting.Packages.LimitTotalOwnerSize {
return ErrQuotaTotalSize
}
}
return nil
}
// GetOrCreateInternalPackageVersion gets or creates an internal package
// Some package types need such internal packages for housekeeping.
func GetOrCreateInternalPackageVersion(ctx context.Context, ownerID int64, packageType packages_model.Type, name, version string) (*packages_model.PackageVersion, error) {
var pv *packages_model.PackageVersion
return pv, db.WithTx(ctx, func(ctx context.Context) error {
p := &packages_model.Package{
OwnerID: ownerID,
Type: packageType,
Name: name,
LowerName: name,
IsInternal: true,
}
var err error
if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
if err != packages_model.ErrDuplicatePackage {
log.Error("Error inserting package: %v", err)
return err
}
}
pv = &packages_model.PackageVersion{
PackageID: p.ID,
CreatorID: ownerID,
Version: version,
LowerVersion: version,
IsInternal: true,
MetadataJSON: "null",
}
if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil {
if err != packages_model.ErrDuplicatePackageVersion {
log.Error("Error inserting package version: %v", err)
return err
}
}
return nil
})
}
// RemovePackageVersionByNameAndVersion deletes a package version and all associated files
func RemovePackageVersionByNameAndVersion(ctx context.Context, doer *user_model.User, pvi *PackageInfo) error {
pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
if err != nil {
return err
}
return RemovePackageVersion(ctx, doer, pv)
}
// RemovePackageVersion deletes the package version and all associated files
func RemovePackageVersion(ctx context.Context, doer *user_model.User, pv *packages_model.PackageVersion) error {
dbCtx, committer, err := db.TxContext(ctx)
if err != nil {
return err
}
defer committer.Close()
pd, err := packages_model.GetPackageDescriptor(dbCtx, pv)
if err != nil {
return err
}
log.Trace("Deleting package: %v", pv.ID)
if err := DeletePackageVersionAndReferences(dbCtx, pv); err != nil {
return err
}
if err := committer.Commit(); err != nil {
return err
}
notify_service.PackageDelete(ctx, doer, pd)
return nil
}
// RemovePackageFileAndVersionIfUnreferenced deletes the package file and the version if there are no referenced files afterwards
func RemovePackageFileAndVersionIfUnreferenced(ctx context.Context, doer *user_model.User, pf *packages_model.PackageFile) error {
var pd *packages_model.PackageDescriptor
if err := db.WithTx(ctx, func(ctx context.Context) error {
if err := DeletePackageFile(ctx, pf); err != nil {
return err
}
has, err := packages_model.HasVersionFileReferences(ctx, pf.VersionID)
if err != nil {
return err
}
if !has {
pv, err := packages_model.GetVersionByID(ctx, pf.VersionID)
if err != nil {
return err
}
pd, err = packages_model.GetPackageDescriptor(ctx, pv)
if err != nil {
return err
}
if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
if pd != nil {
notify_service.PackageDelete(ctx, doer, pd)
}
return nil
}
// DeletePackageVersionAndReferences deletes the package version and its properties and files
func DeletePackageVersionAndReferences(ctx context.Context, pv *packages_model.PackageVersion) error {
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil {
return err
}
pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
if err != nil {
return err
}
for _, pf := range pfs {
if err := DeletePackageFile(ctx, pf); err != nil {
return err
}
}
return packages_model.DeleteVersionByID(ctx, pv.ID)
}
// DeletePackageFile deletes the package file and its properties
func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) error {
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
return err
}
return packages_model.DeleteFileByID(ctx, pf.ID)
}
// GetFileStreamByPackageNameAndVersion returns the content of the specific package file
func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey)
pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
if err != nil {
if err == packages_model.ErrPackageNotExist {
return nil, nil, nil, err
}
log.Error("Error getting package: %v", err)
return nil, nil, nil, err
}
return GetFileStreamByPackageVersion(ctx, pv, pfi)
}
// GetFileStreamByPackageVersion returns the content of the specific package file
func GetFileStreamByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfi.Filename, pfi.CompositeKey)
if err != nil {
return nil, nil, nil, err
}
return GetPackageFileStream(ctx, pf)
}
// GetPackageFileStream returns the content of the specific package file
func GetPackageFileStream(ctx context.Context, pf *packages_model.PackageFile) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
if err != nil {
return nil, nil, nil, err
}
return GetPackageBlobStream(ctx, pf, pb)
}
// GetPackageBlobStream returns the content of the specific package blob
// If the storage supports direct serving and it's enabled, only the direct serving url is returned.
func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
key := packages_module.BlobHash256Key(pb.HashSHA256)
cs := packages_module.NewContentStore()
var s io.ReadSeekCloser
var u *url.URL
var err error
if cs.ShouldServeDirect() {
u, err = cs.GetServeDirectURL(key, pf.Name)
if err != nil && !errors.Is(err, storage.ErrURLNotSupported) {
log.Error("Error getting serve direct url: %v", err)
}
}
if u == nil {
s, err = cs.Get(key)
}
if err == nil {
if pf.IsLead {
if err := packages_model.IncrementDownloadCounter(ctx, pf.VersionID); err != nil {
log.Error("Error incrementing download counter: %v", err)
}
}
}
return s, u, pf, err
}
// RemoveAllPackages for User
func RemoveAllPackages(ctx context.Context, userID int64) (int, error) {
count := 0
for {
pkgVersions, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
Paginator: &db.ListOptions{
PageSize: repo_model.RepositoryListDefaultPageSize,
Page: 1,
},
OwnerID: userID,
IsInternal: util.OptionalBoolNone,
})
if err != nil {
return count, fmt.Errorf("GetOwnedPackages[%d]: %w", userID, err)
}
if len(pkgVersions) == 0 {
break
}
for _, pv := range pkgVersions {
if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
return count, fmt.Errorf("unable to delete package %d:%s[%d]. Error: %w", pv.PackageID, pv.Version, pv.ID, err)
}
count++
}
}
return count, nil
}