From 8994590ce797177ed0e2683bdb8e5a4fae86858c Mon Sep 17 00:00:00 2001 From: Takwa Date: Mon, 19 Oct 2026 16:39:22 +0200 Subject: [PATCH] Give the content kubo headroom for leaked inbound connection slots Every content node had all 4096 inbound connection slots of kubo's resource manager (System.ConnsInbound) in use since its start on 2026-10-14, and refused new peers: 2.4M to 3.1M refused connections per box. The connection manager was not at fault. It counted 277 to 359 connections per box, under HighWater 600, so it had nothing to trim. The slots were leaked reservations, not connections. On 2026-10-19 each box had about 150 live inbound connections (swarm opened minus closed, and the same number of TCP sockets on :4101 in the kernel), against 4096 reserved inbound slots and about 4100 reserved FD slots, with nothing in the handshake (transient) scope. About 3,950 inbound slots, each also holding an FD slot, belong to no connection, so the leak is on the TCP upgrade path of kubo 0.43.1 / go-libp2p v0.49. The bootstrappers leak the same way: 5,400 to 9,000 excess inbound slots each since their restart on 2026-10-16. 4096 is kubo's own choice: it sets System.ConnsInbound to the largest of MaxMemory in MiB (4GiB = 4096), 2 x ConnMgr.HighWater (1200) and a floor, which is why it is not 2 x HighWater here as it was for the bootstrappers. The leak cannot be stopped from the config, and a restart clears it. So: - set System.ConnsInbound to 65536 in libp2p-resource-limit-overrides.json, the file in the repo that replaced Swarm.ResourceMgr.Limits; the init container writes it on every start, from the kubo-config Secret - raise MaxFileDescriptors and IPFS_FD_MAX to 131072, because each leaked slot also holds an FD slot - keep ConnMgr at 600/200, which fits the real load - after a deploy, fail unless the running node reports the 65536 limit A leaked slot holds no socket and no memory, so the higher limit is headroom until the next restart, not extra load. Co-Authored-By: Claude Opus 5.5 --- k8s/content/deployment.yaml | 7 +++- roles/content/files/kubo-config.json | 2 +- .../libp2p-resource-limit-overrides.json | 5 +++ roles/content/tasks/main.yml | 34 +++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 roles/content/files/libp2p-resource-limit-overrides.json diff --git a/k8s/content/deployment.yaml b/k8s/content/deployment.yaml index 0cdaf55..a008ae9 100644 --- a/k8s/content/deployment.yaml +++ b/k8s/content/deployment.yaml @@ -80,12 +80,16 @@ spec: ipfs repo migrate ipfs config replace - < /config/config-nokey.json fi + # Resource manager limits that kubo would otherwise derive (see + # roles/content); kubo reads this file from the repo at start. + cat /config/libp2p-resource-limit-overrides.json > "$IPFS_PATH/libp2p-resource-limit-overrides.json" actual="$(ipfs config Identity.PeerID)" if [ "$actual" != "$expected" ]; then echo "IDENTITY MISMATCH: repo has $actual, expected $expected - refusing to start" >&2 exit 1 fi echo "identity OK: $actual; Provide.Strategy=$(ipfs config Provide.Strategy)" + echo "resource manager overrides: $(tr -d ' \n' < "$IPFS_PATH/libp2p-resource-limit-overrides.json")" env: - name: IPFS_PATH value: /repo/ipfs @@ -116,8 +120,9 @@ spec: value: /repo/ipfs - name: HOME value: /tmp + # Matches Swarm.ResourceMgr.MaxFileDescriptors (roles/content). - name: IPFS_FD_MAX - value: "65536" + value: "131072" # Soft heap target below the 8Gi limit. - name: GOMEMLIMIT value: 6GiB diff --git a/roles/content/files/kubo-config.json b/roles/content/files/kubo-config.json index 7db558c..8693f3b 100644 --- a/roles/content/files/kubo-config.json +++ b/roles/content/files/kubo-config.json @@ -204,7 +204,7 @@ }, "ResourceMgr": { "Enabled": true, - "MaxFileDescriptors": 65536, + "MaxFileDescriptors": 131072, "MaxMemory": "4GiB" }, "Transports": { diff --git a/roles/content/files/libp2p-resource-limit-overrides.json b/roles/content/files/libp2p-resource-limit-overrides.json new file mode 100644 index 0000000..27d2d8c --- /dev/null +++ b/roles/content/files/libp2p-resource-limit-overrides.json @@ -0,0 +1,5 @@ +{ + "System": { + "ConnsInbound": 65536 + } +} diff --git a/roles/content/tasks/main.yml b/roles/content/tasks/main.yml index ec3ee71..e772fcf 100644 --- a/roles/content/tasks/main.yml +++ b/roles/content/tasks/main.yml @@ -139,6 +139,18 @@ # off - no dependency on Shipyard-operated services. # Addresses.Gateway null - no HTTP gateway. API on 127.0.0.1:5021, swarm 4101. # Datastore.StorageMax 90GB - only the GC threshold (GC is not enabled). +# Swarm.ConnMgr 600/200 - what this node really uses (about 300 conns). +# Resource manager: System.ConnsInbound 65536, set in +# files/libp2p-resource-limit-overrides.json (kubo dropped +# Swarm.ResourceMgr.Limits; this file in the repo replaced it). Left to +# kubo it is the larger of MaxMemory in MiB (4096) and 2 x HighWater +# (1200). kubo 0.43.1 leaks inbound TCP connection reservations: on +# 2026-10-19 every box held all 4096 slots and ~4100 FD slots for ~150 +# live inbound conns, so it refused new peers while ConnMgr, below +# HighWater, had nothing to trim. A leaked slot is accounting only (no +# socket, no memory) and a restart clears them, so the limit is headroom +# until restarts, not capacity. MaxFileDescriptors 131072 (and +# IPFS_FD_MAX) for the same reason: each leaked slot also holds an FD. - name: Build this box's kubo config vars: base: "{{ lookup('ansible.builtin.file', role_path ~ '/files/kubo-config.json') | from_json }}" @@ -210,6 +222,9 @@ {{ content_kubo_config | combine({'Identity': {'PeerID': content_kubo_peer_id}}) | to_nice_json | b64encode }} peer-id: "{{ content_kubo_peer_id | b64encode }}" + libp2p-resource-limit-overrides.json: >- + {{ lookup('ansible.builtin.file', role_path ~ '/files/libp2p-resource-limit-overrides.json') + | from_json | to_nice_json | b64encode }} no_log: true register: content_kubo_secret # On a first --check the namespace is only simulated and a dry-run secret @@ -337,6 +352,25 @@ failed_when: content_kubo_config_diff.rc != 0 when: not ansible_check_mode +# The overrides file is read only at start; prove the running node uses it. +- name: Read the running kubo's resource manager limits + ansible.builtin.command: + argv: [k3s, kubectl, --namespace=content, exec, deploy/content, -c, kubo, --, ipfs, swarm, resources, --enc=json] + register: content_kubo_resources + changed_when: false + when: not ansible_check_mode + +- name: Fail unless the resource manager limits match the overrides + vars: + want: "{{ lookup('ansible.builtin.file', role_path ~ '/files/libp2p-resource-limit-overrides.json') | from_json }}" + live: "{{ (content_kubo_resources.stdout | from_json).System }}" + ansible.builtin.assert: + that: + - live.ConnsInbound | int == want.System.ConnsInbound | int + fail_msg: "System.ConnsInbound is {{ live.ConnsInbound }}, want {{ want.System.ConnsInbound }}" + quiet: true + when: not ansible_check_mode + - name: Fail unless the cluster peer runs the vaulted identity and sees kubo vars: peer: "{{ content_cluster_id.json }}"