Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions internal/bootstrap/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ func InitConfig() {
if !conf.Conf.Force {
confFromEnv()
}
conf.CachePolicy = conf.Conf.CachePolicy
log.Infof("cache policy: %s", conf.CachePolicy)

if conf.Conf.MaxConcurrency > math.MaxInt32 {
net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: math.MaxInt32}
Expand Down
62 changes: 62 additions & 0 deletions internal/cache/policy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package cache

import (
"fmt"
"strings"
)

type Policy string

const (
// PolicyInherit is only used by per-instance overrides. It is not a
// user-selectable cache policy.
PolicyInherit Policy = ""
PolicyAuto Policy = "auto"
PolicyMemory Policy = "memory"
PolicyDisk Policy = "disk"
)

func ParsePolicy(value string) (Policy, error) {
policy := Policy(strings.ToLower(strings.TrimSpace(value)))
if policy == PolicyInherit {
return PolicyAuto, nil
}
if !policy.IsConcrete() {
return PolicyInherit, fmt.Errorf("invalid cache policy %q: expected auto, memory, or disk", value)
}
return policy, nil
}

func ResolvePolicy(override, fallback Policy) (Policy, error) {
policy := override
if policy == PolicyInherit {
policy = fallback
}
if !policy.IsConcrete() {
return PolicyInherit, fmt.Errorf("invalid cache policy %q: expected auto, memory, or disk", policy)
}
return policy, nil
}

func (p Policy) IsConcrete() bool {
return p == PolicyAuto || p == PolicyMemory || p == PolicyDisk
}

func (p Policy) MarshalText() ([]byte, error) {
if p == PolicyInherit {
return []byte{}, nil
}
if !p.IsConcrete() {
return nil, fmt.Errorf("invalid cache policy %q", p)
}
return []byte(p), nil
}

func (p *Policy) UnmarshalText(text []byte) error {
policy, err := ParsePolicy(string(text))
if err != nil {
return err
}
*p = policy
return nil
}
82 changes: 82 additions & 0 deletions internal/cache/policy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package cache

import (
"encoding/json"
"testing"

"github.com/caarlos0/env/v9"
)

func TestParsePolicy(t *testing.T) {
tests := []struct {
input string
want Policy
}{
{"", PolicyAuto},
{"auto", PolicyAuto},
{" MEMORY ", PolicyMemory},
{"Disk", PolicyDisk},
}
for _, tt := range tests {
got, err := ParsePolicy(tt.input)
if err != nil {
t.Fatalf("ParsePolicy(%q) error = %v", tt.input, err)
}
if got != tt.want {
t.Errorf("ParsePolicy(%q) = %q, want %q", tt.input, got, tt.want)
}
}
if _, err := ParsePolicy("hybrid"); err == nil {
t.Fatal("ParsePolicy() expected an error for an invalid policy")
}
}

func TestPolicyEnvironment(t *testing.T) {
t.Setenv("OPENLIST_TEST_CACHE_POLICY", " Disk ")
var cfg struct {
Policy Policy `env:"CACHE_POLICY"`
}
if err := env.ParseWithOptions(&cfg, env.Options{Prefix: "OPENLIST_TEST_"}); err != nil {
t.Fatalf("env.ParseWithOptions() error = %v", err)
}
if cfg.Policy != PolicyDisk {
t.Fatalf("environment policy = %q, want disk", cfg.Policy)
}
}

func TestResolvePolicy(t *testing.T) {
got, err := ResolvePolicy(PolicyInherit, PolicyDisk)
if err != nil || got != PolicyDisk {
t.Fatalf("ResolvePolicy(inherit, disk) = %q, %v", got, err)
}
got, err = ResolvePolicy(PolicyMemory, PolicyDisk)
if err != nil || got != PolicyMemory {
t.Fatalf("ResolvePolicy(memory, disk) = %q, %v", got, err)
}
if _, err := ResolvePolicy(PolicyInherit, Policy("invalid")); err == nil {
t.Fatal("ResolvePolicy() expected an error for an invalid fallback")
}
}

func TestPolicyJSON(t *testing.T) {
type config struct {
Policy Policy `json:"cache_policy"`
}
var cfg config
if err := json.Unmarshal([]byte(`{"cache_policy":" MEMORY "}`), &cfg); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if cfg.Policy != PolicyMemory {
t.Fatalf("json.Unmarshal() policy = %q, want memory", cfg.Policy)
}
b, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if string(b) != `{"cache_policy":"memory"}` {
t.Fatalf("json.Marshal() = %s", b)
}
if err := json.Unmarshal([]byte(`{"cache_policy":"invalid"}`), &cfg); err == nil {
t.Fatal("json.Unmarshal() expected an error for an invalid policy")
}
}
57 changes: 30 additions & 27 deletions internal/conf/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package conf
import (
"path/filepath"

"github.com/OpenListTeam/OpenList/v4/internal/cache"
"github.com/OpenListTeam/OpenList/v4/pkg/utils/random"
)

Expand Down Expand Up @@ -112,33 +113,34 @@ type MCP struct {
}

type Config struct {
Force bool `json:"force" env:"FORCE"`
SiteURL string `json:"site_url" env:"SITE_URL"`
Cdn string `json:"cdn" env:"CDN"`
JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"`
TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"`
Database Database `json:"database" envPrefix:"DB_"`
Meilisearch Meilisearch `json:"meilisearch" envPrefix:"MEILISEARCH_"`
Scheme Scheme `json:"scheme"`
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"`
DistDir string `json:"dist_dir"`
Log LogConfig `json:"log" envPrefix:"LOG_"`
DelayedStart int `json:"delayed_start" env:"DELAYED_START"`
AutoMemoryLimit int `json:"auto_memory_limit" env:"AUTO_MEMORY_LIMIT"`
MinFreeMemory int `json:"min_free_memory" env:"MIN_FREE_MEMORY"`
MaxBlockLimit int `json:"max_block_limit" env:"MAX_BLOCK_LIMIT"`
MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"`
MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"`
TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"`
Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"`
Cors Cors `json:"cors" envPrefix:"CORS_"`
S3 S3 `json:"s3" envPrefix:"S3_"`
FTP FTP `json:"ftp" envPrefix:"FTP_"`
SFTP SFTP `json:"sftp" envPrefix:"SFTP_"`
MCP MCP `json:"mcp" envPrefix:"MCP_"`
LastLaunchedVersion string `json:"last_launched_version"`
ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"`
Force bool `json:"force" env:"FORCE"`
SiteURL string `json:"site_url" env:"SITE_URL"`
Cdn string `json:"cdn" env:"CDN"`
JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"`
TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"`
Database Database `json:"database" envPrefix:"DB_"`
Meilisearch Meilisearch `json:"meilisearch" envPrefix:"MEILISEARCH_"`
Scheme Scheme `json:"scheme"`
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"`
DistDir string `json:"dist_dir"`
Log LogConfig `json:"log" envPrefix:"LOG_"`
DelayedStart int `json:"delayed_start" env:"DELAYED_START"`
CachePolicy cache.Policy `json:"cache_policy" env:"CACHE_POLICY"`
AutoMemoryLimit int `json:"auto_memory_limit" env:"AUTO_MEMORY_LIMIT"`
MinFreeMemory int `json:"min_free_memory" env:"MIN_FREE_MEMORY"`
MaxBlockLimit int `json:"max_block_limit" env:"MAX_BLOCK_LIMIT"`
MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"`
MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"`
TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"`
Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"`
Cors Cors `json:"cors" envPrefix:"CORS_"`
S3 S3 `json:"s3" envPrefix:"S3_"`
FTP FTP `json:"ftp" envPrefix:"FTP_"`
SFTP SFTP `json:"sftp" envPrefix:"SFTP_"`
MCP MCP `json:"mcp" envPrefix:"MCP_"`
LastLaunchedVersion string `json:"last_launched_version"`
ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"`
}

func DefaultConfig(dataDir string) *Config {
Expand Down Expand Up @@ -185,6 +187,7 @@ func DefaultConfig(dataDir string) *Config {
},
},
},
CachePolicy: cache.PolicyAuto,
AutoMemoryLimit: 4,
MaxConnections: 0,
MaxConcurrency: 64,
Expand Down
21 changes: 21 additions & 0 deletions internal/conf/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package conf

import (
"encoding/json"
"testing"

"github.com/OpenListTeam/OpenList/v4/internal/cache"
)

func TestDefaultCachePolicy(t *testing.T) {
cfg := DefaultConfig(t.TempDir())
if got := cfg.CachePolicy; got != cache.PolicyAuto {
t.Fatalf("default cache policy = %q, want auto", got)
}
if err := json.Unmarshal([]byte(`{}`), cfg); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if got := cfg.CachePolicy; got != cache.PolicyAuto {
t.Fatalf("cache policy after loading an old config = %q, want auto", got)
}
}
7 changes: 5 additions & 2 deletions internal/conf/var.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"net/url"
"regexp"
"sync"

"github.com/OpenListTeam/OpenList/v4/internal/cache"
)

var (
Expand All @@ -25,10 +27,11 @@ var FilenameCharMap = make(map[string]string)
var PrivacyReg []*regexp.Regexp

var (
CachePolicy cache.Policy = cache.PolicyAuto
// 在HybridCache中使用[]byte缓存数据流的限制,内存为Go自动管理,直到GC
AutoMemoryLimit uint64 = 4 * 1024 * 1024
// 最小空闲内存,当内存不足时,HybridCache会回退到文件缓存
// 如果为0,HybridCache会使用文件缓存,不占用内存。
// 最小空闲内存,当内存不足时,auto策略会回退到文件缓存
// 如果为0,auto策略会使用文件缓存,不占用内存。
MinFreeMemory uint64 = 16 * 1024 * 1024
// 限制HybridCache手动管理内存单次的扩容大小,超过该阈值将分多次扩容。
// MinFreeMemory大于0时,也限制 Downloader 的PartSize
Expand Down
Loading