From 54e45a3f5d4b97256aeb962a75870f6d62b65df3 Mon Sep 17 00:00:00 2001 From: Taylor McKinnon Date: Wed, 4 Feb 2026 13:56:37 -0800 Subject: [PATCH] impr(WKBCH-23): Add rate limit support --- cmd/config.go | 52 +++++++++++++++++--- cmd/configure.go | 13 ++--- cmd/util.go | 7 ++- go.mod | 12 ++++- go.sum | 41 ++++++++++++++- templates/cloudserver/Dockerfile.setup | 11 +++++ templates/cloudserver/config-v7.json | 40 ++++++++++++++- templates/cloudserver/config-v9.json | 38 +++++++++++++- templates/cloudserver/create-service-user.sh | 25 ++++++++++ templates/global/docker-compose.yaml | 18 +++++++ templates/global/values.yaml | 3 ++ templates/scuba/create-service-user.sh | 4 +- 12 files changed, 243 insertions(+), 21 deletions(-) create mode 100644 templates/cloudserver/Dockerfile.setup create mode 100644 templates/cloudserver/create-service-user.sh diff --git a/cmd/config.go b/cmd/config.go index 65237ba..8e14490 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -67,14 +67,15 @@ type GlobalConfig struct { } type FeatureConfig struct { - Scuba ScubaFeatureConfig `yaml:"scuba"` - BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"` + Scuba ScubaFeatureConfig `yaml:"scuba"` + BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"` CrossRegionReplication CrossRegionReplicationFeatureConfig `yaml:"cross_region_replication"` - Utapi UtapiFeatureConfig `yaml:"utapi"` - Migration MigrationFeatureConfig `yaml:"migration"` - AccessLogging AccessLoggingFeatureConfig `yaml:"access_logging"` - S3Frontend S3FrontendFeatureConfig `yaml:"s3_frontend"` - Lifecycle LifecycleFeatureConfig `yaml:"lifecycle"` + Utapi UtapiFeatureConfig `yaml:"utapi"` + Migration MigrationFeatureConfig `yaml:"migration"` + AccessLogging AccessLoggingFeatureConfig `yaml:"access_logging"` + S3Frontend S3FrontendFeatureConfig `yaml:"s3_frontend"` + Lifecycle LifecycleFeatureConfig `yaml:"lifecycle"` + RateLimiting RateLimitingFeatureConfig `yaml:"rate_limiting"` } type S3FrontendFeatureConfig struct { @@ -245,6 +246,29 @@ type LifecycleFeatureConfig struct { Enabled bool `yaml:"enabled"` } +type RateLimitingFeatureConfig struct { + Enabled bool `yaml:"enabled"` + Bucket RateLimitingDefaultLimits `yaml:"bucket"` + Account RateLimitingDefaultLimits `yaml:"account"` + Error RateLimitingErrorConfig `yaml:"error"` +} + +type RateLimitingDefaultLimits struct { + RequestsPerSecond *RateLimitingLimitConfig `yaml:"requests_per_second"` + ConfigCacheTTL int `yaml:"config_cache_ttl"` +} + +type RateLimitingLimitConfig struct { + Limit int `yaml:"limit"` + BurstCapacity int `yaml:"burst_capacity"` +} + +type RateLimitingErrorConfig struct { + StatusCode int `yaml:"status_code"` + Code string `yaml:"code"` + Message string `yaml:"message"` +} + func DefaultEnvironmentConfig() EnvironmentConfig { return EnvironmentConfig{ Global: GlobalConfig{ @@ -275,6 +299,20 @@ func DefaultEnvironmentConfig() EnvironmentConfig { Lifecycle: LifecycleFeatureConfig{ Enabled: false, }, + RateLimiting: RateLimitingFeatureConfig{ + Enabled: false, + Bucket: RateLimitingDefaultLimits{ + ConfigCacheTTL: 30000, + }, + Account: RateLimitingDefaultLimits{ + ConfigCacheTTL: 30000, + }, + Error: RateLimitingErrorConfig{ + StatusCode: 429, + Code: "SlowDown", + Message: "Rate limit exceeded. Please try again later.", + }, + }, }, Cloudserver: CloudserverConfig{}, S3Metadata: MetadataConfig{ diff --git a/cmd/configure.go b/cmd/configure.go index 6c3dcc0..fae91a6 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -130,12 +130,13 @@ func generateCloudserverConfig(cfg EnvironmentConfig, path string) error { return err } - return renderTemplateToFile( - getTemplates(), - "templates/cloudserver/locationConfig.json", - cfg, - filepath.Join(path, "cloudserver", "locationConfig.json"), - ) + templates := []string{ + "locationConfig.json", + "create-service-user.sh", + "Dockerfile.setup", + } + + return renderTemplates(cfg, "templates/cloudserver", filepath.Join(path, "cloudserver"), templates) } func generateBackbeatConfig(cfg EnvironmentConfig, path string) error { diff --git a/cmd/util.go b/cmd/util.go index d8cae7f..07b3b2b 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -11,6 +11,7 @@ import ( "strings" "text/template" + "github.com/Masterminds/sprig/v3" "github.com/hashicorp/go-multierror" "github.com/scality/workbench" @@ -25,7 +26,7 @@ func getTemplates() fs.FS { } func templateFile(templates fs.FS, path string, data any) ([]byte, error) { - tmpl, err := template.ParseFS(templates, path) + tmpl, err := template.New(filepath.Base(path)).Funcs(sprig.FuncMap()).ParseFS(templates, path) if err != nil { return nil, err } @@ -100,6 +101,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string { profiles = append(profiles, "feature-lifecycle") } + if cfg.Features.RateLimiting.Enabled { + profiles = append(profiles, "feature-rate-limiting") + } + return profiles } diff --git a/go.mod b/go.mod index f0a5629..f3f038b 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.24.3 require ( dario.cat/mergo v1.0.2 + github.com/Masterminds/sprig/v3 v3.3.0 github.com/alecthomas/kong v1.11.0 github.com/hashicorp/go-multierror v1.1.1 github.com/rs/zerolog v1.34.0 @@ -11,8 +12,17 @@ require ( ) require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - golang.org/x/sys v0.12.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/spf13/cast v1.7.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/sys v0.23.0 // indirect ) diff --git a/go.sum b/go.sum index a90b013..f470c35 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/kong v1.11.0 h1:y++1gI7jf8O7G7l4LZo5ASFhrhJvzc+WgF/arranEmM= @@ -7,27 +13,60 @@ github.com/alecthomas/kong v1.11.0/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruP github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/templates/cloudserver/Dockerfile.setup b/templates/cloudserver/Dockerfile.setup new file mode 100644 index 0000000..2628ea4 --- /dev/null +++ b/templates/cloudserver/Dockerfile.setup @@ -0,0 +1,11 @@ +ARG BASE_IMAGE + +FROM $BASE_IMAGE + +USER root + +RUN apt-get update && apt-get install -y jq curl + +COPY --chmod=755 create-service-user.sh /opt/ + +CMD ["/opt/create-service-user.sh"] diff --git a/templates/cloudserver/config-v7.json b/templates/cloudserver/config-v7.json index 306b974..b180d90 100644 --- a/templates/cloudserver/config-v7.json +++ b/templates/cloudserver/config-v7.json @@ -98,11 +98,13 @@ {{ else }} "bucketNotificationDestinations": [], {{ end }} - {{ if .Features.Utapi.Enabled }} + {{ if or .Features.Utapi.Enabled .Features.RateLimiting.Enabled }} "localCache": { "host": "localhost", "port": 6379 }, + {{ end }} + {{ if .Features.Utapi.Enabled }} "utapi": { "host": "localhost", "port": 8100, @@ -112,6 +114,40 @@ "port": 6379 } }, - {{ end }} + {{- end }} + {{- if .Features.RateLimiting.Enabled }} + "rateLimiting": { + "enabled": {{ .Features.RateLimiting.Enabled }}, + "serviceUserArn": "arn:aws:iam::000000000000:user/scality-internal/service-rate-limit-user", + "nodes": 1, + "bucket": { + {{- if .Features.RateLimiting.Bucket.RequestsPerSecond }} + "defaultConfig": { + "requestsPerSecond": { + "limit": {{ .Features.RateLimiting.Bucket.RequestsPerSecond.Limit }}, + "burstCapacity": {{ .Features.RateLimiting.Bucket.RequestsPerSecond.BurstCapacity }} + } + }, + {{- end }} + "configCacheTTL": {{ .Features.RateLimiting.Bucket.ConfigCacheTTL }} + }, + "account": { + {{- if .Features.RateLimiting.Account.RequestsPerSecond }} + "defaultConfig": { + "requestsPerSecond": { + "limit": {{ .Features.RateLimiting.Account.RequestsPerSecond.Limit }}, + "burstCapacity": {{ .Features.RateLimiting.Account.RequestsPerSecond.BurstCapacity }} + } + }, + {{- end }} + "configCacheTTL": {{ .Features.RateLimiting.Account.ConfigCacheTTL }} + }, + "error": { + "statusCode": {{ .Features.RateLimiting.Error.StatusCode }}, + "code": {{ .Features.RateLimiting.Error.Code | toJson }}, + "message": {{ .Features.RateLimiting.Error.Message | toJson }} + } + }, + {{- end }} "testingMode": true } diff --git a/templates/cloudserver/config-v9.json b/templates/cloudserver/config-v9.json index 23e5532..3654eb8 100644 --- a/templates/cloudserver/config-v9.json +++ b/templates/cloudserver/config-v9.json @@ -150,11 +150,13 @@ {{ else }} "bucketNotificationDestinations": [], {{ end }} - {{ if .Features.Utapi.Enabled }} + {{ if or .Features.Utapi.Enabled .Features.RateLimiting.Enabled }} "localCache": { "host": "localhost", "port": 6379 }, + {{ end }} + {{ if .Features.Utapi.Enabled }} "utapi": { "host": "localhost", "port": 8100, @@ -195,5 +197,39 @@ "retryReopenDelayMS": 1000, "checkFileRotationIntervalMS": 10000 }, + {{- if .Features.RateLimiting.Enabled }} + "rateLimiting": { + "enabled": {{ .Features.RateLimiting.Enabled }}, + "serviceUserArn": "arn:aws:iam::000000000000:user/scality-internal/service-rate-limit-user", + "nodes": 1, + "bucket": { + {{- if .Features.RateLimiting.Bucket.RequestsPerSecond }} + "defaultConfig": { + "requestsPerSecond": { + "limit": {{ .Features.RateLimiting.Bucket.RequestsPerSecond.Limit }}, + "burstCapacity": {{ .Features.RateLimiting.Bucket.RequestsPerSecond.BurstCapacity }} + } + }, + {{- end }} + "configCacheTTL": {{ .Features.RateLimiting.Bucket.ConfigCacheTTL }} + }, + "account": { + {{- if .Features.RateLimiting.Account.RequestsPerSecond }} + "defaultConfig": { + "requestsPerSecond": { + "limit": {{ .Features.RateLimiting.Account.RequestsPerSecond.Limit }}, + "burstCapacity": {{ .Features.RateLimiting.Account.RequestsPerSecond.BurstCapacity }} + } + }, + {{- end }} + "configCacheTTL": {{ .Features.RateLimiting.Account.ConfigCacheTTL }} + }, + "error": { + "statusCode": {{ .Features.RateLimiting.Error.StatusCode }}, + "code": {{ .Features.RateLimiting.Error.Code | toJson }}, + "message": {{ .Features.RateLimiting.Error.Message | toJson }} + } + }, + {{- end }} "testingMode": true } diff --git a/templates/cloudserver/create-service-user.sh b/templates/cloudserver/create-service-user.sh new file mode 100644 index 0000000..994b79d --- /dev/null +++ b/templates/cloudserver/create-service-user.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env sh +set -xe + +MANAGEMENT_ACCESS_KEY=$(jq -r '.accessKey' /secrets/management-creds.json) +MANAGEMENT_SECRET_KEY=$(jq -r '.secretKey' /secrets/management-creds.json) + +# === Create rate-limit service user === +echo "[setup] Creating rate-limit service user..." +SERVICE_CREDS_JSON=$(AWS_ACCESS_KEY_ID="$MANAGEMENT_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$MANAGEMENT_SECRET_KEY" \ + AWS_REGION="us-east-1" \ + ./bin/ensureServiceUser apply service-rate-limit-user --iam-endpoint http://127.0.0.1:8600) + +SERVICE_ACCESS_KEY=$(echo "$SERVICE_CREDS_JSON" | jq -r '.data.AccessKeyId') +SERVICE_SECRET_KEY=$(echo "$SERVICE_CREDS_JSON" | jq -r '.data.SecretAccessKey') + +echo "[setup] rate-limit service user credentials:" +echo "SERVICE_ACCESS_KEY=$SERVICE_ACCESS_KEY" +echo "SERVICE_SECRET_KEY=$SERVICE_SECRET_KEY" +echo + +# === Update rate-limit-service-creds.json === +echo "[setup] Updating rate-limit-service-creds.json with service user credentials..." +jq --null-input --arg ak "$SERVICE_ACCESS_KEY" --arg sk "$SERVICE_SECRET_KEY" \ + '{accessKey: $ak, secretKey: $sk}' > /secrets/rate-limit-service-creds.json diff --git a/templates/global/docker-compose.yaml b/templates/global/docker-compose.yaml index bf5b032..d296f3d 100644 --- a/templates/global/docker-compose.yaml +++ b/templates/global/docker-compose.yaml @@ -70,6 +70,23 @@ services: profiles: - base + setup-rate-limiting-svc-user: + build: + context: ./config/cloudserver + dockerfile: Dockerfile.setup + args: + # reuse scuba image since cloudserver does not contain the create-service-user script + BASE_IMAGE: ${SCUBA_IMAGE} + container_name: workbench-setup-cloudserver + network_mode: host + depends_on: + setup-vault: + condition: service_completed_successfully + volumes: + - ./config/vault/:/secrets + profiles: + - feature-rate-limiting + s3-frontend: image: ${NGINX_IMAGE} container_name: workbench-s3-frontend @@ -211,6 +228,7 @@ services: - feature-utapi - feature-migration - feature-lifecycle + - feature-rate-limiting zookeeper: build: diff --git a/templates/global/values.yaml b/templates/global/values.yaml index ccf9d85..b464875 100644 --- a/templates/global/values.yaml +++ b/templates/global/values.yaml @@ -31,6 +31,9 @@ features: lifecycle: enabled: false + rate_limiting: + enabled: false + cloudserver: image: ghcr.io/scality/cloudserver:9.2.22 diff --git a/templates/scuba/create-service-user.sh b/templates/scuba/create-service-user.sh index 45d5c99..05df884 100644 --- a/templates/scuba/create-service-user.sh +++ b/templates/scuba/create-service-user.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -set -x +set -xe MANAGEMENT_ACCESS_KEY=$(jq -r '.accessKey' /secrets/management-creds.json) MANAGEMENT_SECRET_KEY=$(jq -r '.secretKey' /secrets/management-creds.json) @@ -8,7 +8,7 @@ MANAGEMENT_SECRET_KEY=$(jq -r '.secretKey' /secrets/management-creds.json) echo "[setup] Creating scuba service user..." SERVICE_CREDS_JSON=$(AWS_ACCESS_KEY_ID="$MANAGEMENT_ACCESS_KEY" \ AWS_SECRET_ACCESS_KEY="$MANAGEMENT_SECRET_KEY" \ - AWS_REGION="$REGION" \ + AWS_REGION="us-east-1" \ ./bin/ensureServiceUser apply service-scuba-user --iam-endpoint http://127.0.0.1:8600) SERVICE_ACCESS_KEY=$(echo "$SERVICE_CREDS_JSON" | jq -r '.data.AccessKeyId')