Azure blob storage support (#30995)

This PR implemented object storages(LFS/Packages/Attachments and etc.)
for Azure Blob Storage. It depends on azure official golang SDK and can
support both the azure blob storage cloud service and azurite mock
server.

Replace #25458
Fix #22527 

- [x] CI Tests
- [x] integration test, MSSQL integration tests will now based on
azureblob
  - [x] unit test 
- [x] CLI Migrate Storage
- [x] Documentation for configuration added

------

TODO (other PRs):
- [ ] Improve performance of `blob download`.

---------

Co-authored-by: yp05327 <576951401@qq.com>
This commit is contained in:
2024-05-30 15:33:50 +08:00
committed by GitHub
parent 015efcd8bf
commit fb7b743bd0
31 changed files with 779 additions and 55 deletions

View File

@ -97,6 +97,44 @@ STORAGE_TYPE = minio
assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath)
}
func Test_getStorageInheritStorageTypeAzureBlob(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = azureblob
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))
assert.EqualValues(t, "azureblob", Packages.Storage.Type)
assert.EqualValues(t, "gitea", Packages.Storage.AzureBlobConfig.Container)
assert.EqualValues(t, "packages/", Packages.Storage.AzureBlobConfig.BasePath)
assert.NoError(t, loadRepoArchiveFrom(cfg))
assert.EqualValues(t, "azureblob", RepoArchive.Storage.Type)
assert.EqualValues(t, "gitea", RepoArchive.Storage.AzureBlobConfig.Container)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath)
assert.NoError(t, loadActionsFrom(cfg))
assert.EqualValues(t, "azureblob", Actions.LogStorage.Type)
assert.EqualValues(t, "gitea", Actions.LogStorage.AzureBlobConfig.Container)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.AzureBlobConfig.BasePath)
assert.EqualValues(t, "azureblob", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "gitea", Actions.ArtifactStorage.AzureBlobConfig.Container)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.AzureBlobConfig.BasePath)
assert.NoError(t, loadAvatarsFrom(cfg))
assert.EqualValues(t, "azureblob", Avatar.Storage.Type)
assert.EqualValues(t, "gitea", Avatar.Storage.AzureBlobConfig.Container)
assert.EqualValues(t, "avatars/", Avatar.Storage.AzureBlobConfig.BasePath)
assert.NoError(t, loadRepoAvatarFrom(cfg))
assert.EqualValues(t, "azureblob", RepoAvatar.Storage.Type)
assert.EqualValues(t, "gitea", RepoAvatar.Storage.AzureBlobConfig.Container)
assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.AzureBlobConfig.BasePath)
}
type testLocalStoragePathCase struct {
loader func(rootCfg ConfigProvider) error
storagePtr **Storage
@ -465,3 +503,77 @@ MINIO_BASE_PATH = /lfs
assert.EqualValues(t, true, LFS.Storage.MinioConfig.UseSSL)
assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath)
}
func Test_getStorageConfiguration29(t *testing.T) {
cfg, err := NewConfigProviderFromData(`
[repo-archive]
STORAGE_TYPE = azureblob
AZURE_BLOB_ACCOUNT_NAME = my_account_name
AZURE_BLOB_ACCOUNT_KEY = my_account_key
`)
assert.NoError(t, err)
// assert.Error(t, loadRepoArchiveFrom(cfg))
// FIXME: this should return error but now ini package's MapTo() doesn't check type
assert.NoError(t, loadRepoArchiveFrom(cfg))
}
func Test_getStorageConfiguration30(t *testing.T) {
cfg, err := NewConfigProviderFromData(`
[storage.repo-archive]
STORAGE_TYPE = azureblob
AZURE_BLOB_ACCOUNT_NAME = my_account_name
AZURE_BLOB_ACCOUNT_KEY = my_account_key
`)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))
assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName)
assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath)
}
func Test_getStorageConfiguration31(t *testing.T) {
cfg, err := NewConfigProviderFromData(`
[storage]
STORAGE_TYPE = azureblob
AZURE_BLOB_ACCOUNT_NAME = my_account_name
AZURE_BLOB_ACCOUNT_KEY = my_account_key
AZURE_BLOB_BASE_PATH = /prefix
`)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))
assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName)
assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey)
assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath)
cfg, err = NewConfigProviderFromData(`
[storage]
STORAGE_TYPE = azureblob
AZURE_BLOB_ACCOUNT_NAME = my_account_name
AZURE_BLOB_ACCOUNT_KEY = my_account_key
AZURE_BLOB_BASE_PATH = /prefix
[lfs]
AZURE_BLOB_BASE_PATH = /lfs
`)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))
assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName)
assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey)
assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath)
cfg, err = NewConfigProviderFromData(`
[storage]
STORAGE_TYPE = azureblob
AZURE_BLOB_ACCOUNT_NAME = my_account_name
AZURE_BLOB_ACCOUNT_KEY = my_account_key
AZURE_BLOB_BASE_PATH = /prefix
[storage.lfs]
AZURE_BLOB_BASE_PATH = /lfs
`)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))
assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName)
assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey)
assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath)
}