diff --git a/k8s/content/deployment.yaml b/k8s/content/deployment.yaml index 0cdaf55..a008ae9 100644 --- a/k8s/content/deployment.yaml +++ b/k8s/content/deployment.yaml @@ -80,12 +80,16 @@ spec: ipfs repo migrate ipfs config replace - < /config/config-nokey.json fi + # Resource manager limits that kubo would otherwise derive (see + # roles/content); kubo reads this file from the repo at start. + cat /config/libp2p-resource-limit-overrides.json > "$IPFS_PATH/libp2p-resource-limit-overrides.json" actual="$(ipfs config Identity.PeerID)" if [ "$actual" != "$expected" ]; then echo "IDENTITY MISMATCH: repo has $actual, expected $expected - refusing to start" >&2 exit 1 fi echo "identity OK: $actual; Provide.Strategy=$(ipfs config Provide.Strategy)" + echo "resource manager overrides: $(tr -d ' \n' < "$IPFS_PATH/libp2p-resource-limit-overrides.json")" env: - name: IPFS_PATH value: /repo/ipfs @@ -116,8 +120,9 @@ spec: value: /repo/ipfs - name: HOME value: /tmp + # Matches Swarm.ResourceMgr.MaxFileDescriptors (roles/content). - name: IPFS_FD_MAX - value: "65536" + value: "131072" # Soft heap target below the 8Gi limit. - name: GOMEMLIMIT value: 6GiB diff --git a/roles/content/files/kubo-config.json b/roles/content/files/kubo-config.json index 7db558c..8693f3b 100644 --- a/roles/content/files/kubo-config.json +++ b/roles/content/files/kubo-config.json @@ -204,7 +204,7 @@ }, "ResourceMgr": { "Enabled": true, - "MaxFileDescriptors": 65536, + "MaxFileDescriptors": 131072, "MaxMemory": "4GiB" }, "Transports": { diff --git a/roles/content/files/libp2p-resource-limit-overrides.json b/roles/content/files/libp2p-resource-limit-overrides.json new file mode 100644 index 0000000..27d2d8c --- /dev/null +++ b/roles/content/files/libp2p-resource-limit-overrides.json @@ -0,0 +1,5 @@ +{ + "System": { + "ConnsInbound": 65536 + } +} diff --git a/roles/content/tasks/main.yml b/roles/content/tasks/main.yml index ec3ee71..e772fcf 100644 --- a/roles/content/tasks/main.yml +++ b/roles/content/tasks/main.yml @@ -139,6 +139,18 @@ # off - no dependency on Shipyard-operated services. # Addresses.Gateway null - no HTTP gateway. API on 127.0.0.1:5021, swarm 4101. # Datastore.StorageMax 90GB - only the GC threshold (GC is not enabled). +# Swarm.ConnMgr 600/200 - what this node really uses (about 300 conns). +# Resource manager: System.ConnsInbound 65536, set in +# files/libp2p-resource-limit-overrides.json (kubo dropped +# Swarm.ResourceMgr.Limits; this file in the repo replaced it). Left to +# kubo it is the larger of MaxMemory in MiB (4096) and 2 x HighWater +# (1200). kubo 0.43.1 leaks inbound TCP connection reservations: on +# 2026-10-19 every box held all 4096 slots and ~4100 FD slots for ~150 +# live inbound conns, so it refused new peers while ConnMgr, below +# HighWater, had nothing to trim. A leaked slot is accounting only (no +# socket, no memory) and a restart clears them, so the limit is headroom +# until restarts, not capacity. MaxFileDescriptors 131072 (and +# IPFS_FD_MAX) for the same reason: each leaked slot also holds an FD. - name: Build this box's kubo config vars: base: "{{ lookup('ansible.builtin.file', role_path ~ '/files/kubo-config.json') | from_json }}" @@ -210,6 +222,9 @@ {{ content_kubo_config | combine({'Identity': {'PeerID': content_kubo_peer_id}}) | to_nice_json | b64encode }} peer-id: "{{ content_kubo_peer_id | b64encode }}" + libp2p-resource-limit-overrides.json: >- + {{ lookup('ansible.builtin.file', role_path ~ '/files/libp2p-resource-limit-overrides.json') + | from_json | to_nice_json | b64encode }} no_log: true register: content_kubo_secret # On a first --check the namespace is only simulated and a dry-run secret @@ -337,6 +352,25 @@ failed_when: content_kubo_config_diff.rc != 0 when: not ansible_check_mode +# The overrides file is read only at start; prove the running node uses it. +- name: Read the running kubo's resource manager limits + ansible.builtin.command: + argv: [k3s, kubectl, --namespace=content, exec, deploy/content, -c, kubo, --, ipfs, swarm, resources, --enc=json] + register: content_kubo_resources + changed_when: false + when: not ansible_check_mode + +- name: Fail unless the resource manager limits match the overrides + vars: + want: "{{ lookup('ansible.builtin.file', role_path ~ '/files/libp2p-resource-limit-overrides.json') | from_json }}" + live: "{{ (content_kubo_resources.stdout | from_json).System }}" + ansible.builtin.assert: + that: + - live.ConnsInbound | int == want.System.ConnsInbound | int + fail_msg: "System.ConnsInbound is {{ live.ConnsInbound }}, want {{ want.System.ConnsInbound }}" + quiet: true + when: not ansible_check_mode + - name: Fail unless the cluster peer runs the vaulted identity and sees kubo vars: peer: "{{ content_cluster_id.json }}"