forgejo/modules/storage/minio.go
Rowan Bohde 73706ae26d
feat: add support for a credentials chain for minio access ()
We wanted to be able to use the IAM role provided by the EC2 instance
metadata in order to access S3 via the Minio configuration. To do this,
a new credentials chain is added that will check the following locations
for credentials when an access key is not provided. In priority order,
they are:

1. MINIO_ prefixed environment variables
2. AWS_ prefixed environment variables
3. a minio credentials file
4. an aws credentials file
5. EC2 instance metadata

(cherry picked from commit c0880e7695346997c6a93f05cd01634cb3ad03ee)

Conflicts:
	docs/content/administration/config-cheat-sheet.en-us.md
	does not exist in Forgejo
2024-06-02 10:14:09 +02:00

310 lines
8.8 KiB
Go

// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package storage
import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
var (
_ ObjectStorage = &MinioStorage{}
quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
)
type minioObject struct {
*minio.Object
}
func (m *minioObject) Stat() (os.FileInfo, error) {
oi, err := m.Object.Stat()
if err != nil {
return nil, convertMinioErr(err)
}
return &minioFileInfo{oi}, nil
}
// MinioStorage returns a minio bucket storage
type MinioStorage struct {
cfg *setting.MinioStorageConfig
ctx context.Context
client *minio.Client
bucket string
basePath string
}
func convertMinioErr(err error) error {
if err == nil {
return nil
}
errResp, ok := err.(minio.ErrorResponse)
if !ok {
return err
}
// Convert two responses to standard analogues
switch errResp.Code {
case "NoSuchKey":
return os.ErrNotExist
case "AccessDenied":
return os.ErrPermission
}
return err
}
var getBucketVersioning = func(ctx context.Context, minioClient *minio.Client, bucket string) error {
_, err := minioClient.GetBucketVersioning(ctx, bucket)
return err
}
// NewMinioStorage returns a minio storage
func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error) {
config := cfg.MinioConfig
if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
}
var lookup minio.BucketLookupType
switch config.BucketLookup {
case "auto", "":
lookup = minio.BucketLookupAuto
case "dns":
lookup = minio.BucketLookupDNS
case "path":
lookup = minio.BucketLookupPath
default:
return nil, fmt.Errorf("invalid minio bucket lookup type %s", config.BucketLookup)
}
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
minioClient, err := minio.New(config.Endpoint, &minio.Options{
Creds: buildMinioCredentials(config, credentials.DefaultIAMRoleEndpoint),
Secure: config.UseSSL,
Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
Region: config.Location,
BucketLookup: lookup,
})
if err != nil {
return nil, convertMinioErr(err)
}
// The GetBucketVersioning is only used for checking whether the Object Storage parameters are generally good. It doesn't need to succeed.
// The assumption is that if the API returns the HTTP code 400, then the parameters could be incorrect.
// Otherwise even if the request itself fails (403, 404, etc), the code should still continue because the parameters seem "good" enough.
// Keep in mind that GetBucketVersioning requires "owner" to really succeed, so it can't be used to check the existence.
// Not using "BucketExists (HeadBucket)" because it doesn't include detailed failure reasons.
err = getBucketVersioning(ctx, minioClient, config.Bucket)
if err != nil {
errResp, ok := err.(minio.ErrorResponse)
if !ok {
return nil, err
}
if errResp.StatusCode == http.StatusBadRequest {
log.Error("S3 storage connection failure at %s:%s with base path %s and region: %s", config.Endpoint, config.Bucket, config.Location, errResp.Message)
return nil, err
}
}
// Check to see if we already own this bucket
exists, err := minioClient.BucketExists(ctx, config.Bucket)
if err != nil {
return nil, convertMinioErr(err)
}
if !exists {
if err := minioClient.MakeBucket(ctx, config.Bucket, minio.MakeBucketOptions{
Region: config.Location,
}); err != nil {
return nil, convertMinioErr(err)
}
}
return &MinioStorage{
cfg: &config,
ctx: ctx,
client: minioClient,
bucket: config.Bucket,
basePath: config.BasePath,
}, nil
}
func (m *MinioStorage) buildMinioPath(p string) string {
p = strings.TrimPrefix(util.PathJoinRelX(m.basePath, p), "/") // object store doesn't use slash for root path
if p == "." {
p = "" // object store doesn't use dot as relative path
}
return p
}
func (m *MinioStorage) buildMinioDirPrefix(p string) string {
// ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo"
p = m.buildMinioPath(p) + "/"
if p == "/" {
p = "" // object store doesn't use slash for root path
}
return p
}
func buildMinioCredentials(config setting.MinioStorageConfig, iamEndpoint string) *credentials.Credentials {
// If static credentials are provided, use those
if config.AccessKeyID != "" {
return credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "")
}
// Otherwise, fallback to a credentials chain for S3 access
chain := []credentials.Provider{
// configure based upon MINIO_ prefixed environment variables
&credentials.EnvMinio{},
// configure based upon AWS_ prefixed environment variables
&credentials.EnvAWS{},
// read credentials from MINIO_SHARED_CREDENTIALS_FILE
// environment variable, or default json config files
&credentials.FileMinioClient{},
// read credentials from AWS_SHARED_CREDENTIALS_FILE
// environment variable, or default credentials file
&credentials.FileAWSCredentials{},
// read IAM role from EC2 metadata endpoint if available
&credentials.IAM{
Endpoint: iamEndpoint,
Client: &http.Client{
Transport: http.DefaultTransport,
},
},
}
return credentials.NewChainCredentials(chain)
}
// Open opens a file
func (m *MinioStorage) Open(path string) (Object, error) {
opts := minio.GetObjectOptions{}
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
if err != nil {
return nil, convertMinioErr(err)
}
return &minioObject{object}, nil
}
// Save saves a file to minio
func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) {
uploadInfo, err := m.client.PutObject(
m.ctx,
m.bucket,
m.buildMinioPath(path),
r,
size,
minio.PutObjectOptions{
ContentType: "application/octet-stream",
// some storages like:
// * https://developers.cloudflare.com/r2/api/s3/api/
// * https://www.backblaze.com/b2/docs/s3_compatible_api.html
// do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum
SendContentMd5: m.cfg.ChecksumAlgorithm == "md5",
},
)
if err != nil {
return 0, convertMinioErr(err)
}
return uploadInfo.Size, nil
}
type minioFileInfo struct {
minio.ObjectInfo
}
func (m minioFileInfo) Name() string {
return path.Base(m.ObjectInfo.Key)
}
func (m minioFileInfo) Size() int64 {
return m.ObjectInfo.Size
}
func (m minioFileInfo) ModTime() time.Time {
return m.LastModified
}
func (m minioFileInfo) IsDir() bool {
return strings.HasSuffix(m.ObjectInfo.Key, "/")
}
func (m minioFileInfo) Mode() os.FileMode {
return os.ModePerm
}
func (m minioFileInfo) Sys() any {
return nil
}
// Stat returns the stat information of the object
func (m *MinioStorage) Stat(path string) (os.FileInfo, error) {
info, err := m.client.StatObject(
m.ctx,
m.bucket,
m.buildMinioPath(path),
minio.StatObjectOptions{},
)
if err != nil {
return nil, convertMinioErr(err)
}
return &minioFileInfo{info}, nil
}
// Delete delete a file
func (m *MinioStorage) Delete(path string) error {
err := m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})
return convertMinioErr(err)
}
// URL gets the redirect URL to a file. The presigned link is valid for 5 minutes.
func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
reqParams := make(url.Values)
// TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we?
reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
return u, convertMinioErr(err)
}
// IterateObjects iterates across the objects in the miniostorage
func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
opts := minio.GetObjectOptions{}
for mObjInfo := range m.client.ListObjects(m.ctx, m.bucket, minio.ListObjectsOptions{
Prefix: m.buildMinioDirPrefix(dirName),
Recursive: true,
}) {
object, err := m.client.GetObject(m.ctx, m.bucket, mObjInfo.Key, opts)
if err != nil {
return convertMinioErr(err)
}
if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
defer object.Close()
return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
}(object, fn); err != nil {
return convertMinioErr(err)
}
}
return nil
}
func init() {
RegisterStorageType(setting.MinioStorageType, NewMinioStorage)
}