From 8f3c2c4ac02b7a7fe279b1ff4bc6a815c6ef636d Mon Sep 17 00:00:00 2001 From: Takwa Date: Mon, 19 Oct 2026 16:52:07 +0200 Subject: [PATCH] Give the bootstrappers headroom for leaked inbound connection slots kubo 0.43.1 leaks inbound TCP/WS connection reservations in the libp2p resource manager, the same leak as the content node (#10). In the three days after the 2026-10-16 restart each bootstrapper gained 5,400-9,000 inbound slots more than its live inbound connections, about 2,000-3,200 a day, so the 64000 limit from #7 (2 x ConnMgr.HighWater) runs out around 2026-10-31. Set System.ConnsInbound to 262144 in libp2p-resource-limit-overrides.json, carried in the kubo-config Secret and written into the repo by the init container on every start, and raise MaxFileDescriptors and IPFS_FD_MAX to 524288, since each leaked slot also holds an FD slot. IPFS_FD_MAX was still 65536 under a 131072 MaxFileDescriptors; they match again. ConnMgr stays at 32000/16000. After a deploy the role fails unless the running node reports both limits. Co-Authored-By: Claude Opus 5.5 --- k8s/bootstrap/deployment.yaml.j2 | 9 +++- roles/bootstrap/files/kubo-config.json | 2 +- .../libp2p-resource-limit-overrides.json | 5 ++ roles/bootstrap/tasks/instance.yml | 48 +++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 roles/bootstrap/files/libp2p-resource-limit-overrides.json diff --git a/k8s/bootstrap/deployment.yaml.j2 b/k8s/bootstrap/deployment.yaml.j2 index a9d5eec..0d4033d 100644 --- a/k8s/bootstrap/deployment.yaml.j2 +++ b/k8s/bootstrap/deployment.yaml.j2 @@ -84,12 +84,16 @@ spec: ipfs repo migrate ipfs config replace - < /config/config-nokey.json fi + # Resource manager limits that kubo would otherwise derive (see + # roles/bootstrap); kubo reads this file from the repo at start. + cat /config/libp2p-resource-limit-overrides.json > "$IPFS_PATH/libp2p-resource-limit-overrides.json" actual="$(ipfs config Identity.PeerID)" if [ "$actual" != "$expected" ]; then echo "IDENTITY MISMATCH: repo has $actual, expected $expected - refusing to start" >&2 exit 1 fi echo "identity OK: $actual" + echo "resource manager overrides: $(tr -d ' \n' < "$IPFS_PATH/libp2p-resource-limit-overrides.json")" env: - name: IPFS_PATH value: /data/ipfs @@ -121,8 +125,11 @@ spec: - name: HOME value: /tmp # kubo raises its own fd soft limit only to 8192 unless told. + # Matches Swarm.ResourceMgr.MaxFileDescriptors (roles/bootstrap), + # so the resource manager refuses a connection before an open + # would fail with EMFILE. - name: IPFS_FD_MAX - value: "65536" + value: "524288" # Soft heap target below the 16Gi limit; the libp2p resource # manager is capped separately at 8GiB in the kubo config. - name: GOMEMLIMIT diff --git a/roles/bootstrap/files/kubo-config.json b/roles/bootstrap/files/kubo-config.json index 3158b5a..330f6ae 100644 --- a/roles/bootstrap/files/kubo-config.json +++ b/roles/bootstrap/files/kubo-config.json @@ -206,7 +206,7 @@ }, "ResourceMgr": { "Enabled": true, - "MaxFileDescriptors": 131072, + "MaxFileDescriptors": 524288, "MaxMemory": "8GiB" }, "Transports": { diff --git a/roles/bootstrap/files/libp2p-resource-limit-overrides.json b/roles/bootstrap/files/libp2p-resource-limit-overrides.json new file mode 100644 index 0000000..79a5b4c --- /dev/null +++ b/roles/bootstrap/files/libp2p-resource-limit-overrides.json @@ -0,0 +1,5 @@ +{ + "System": { + "ConnsInbound": 262144 + } +} diff --git a/roles/bootstrap/tasks/instance.yml b/roles/bootstrap/tasks/instance.yml index dd826b9..50aef79 100644 --- a/roles/bootstrap/tasks/instance.yml +++ b/roles/bootstrap/tasks/instance.yml @@ -127,6 +127,23 @@ # kubo config. Base settings are committed in files/kubo-config.json (no # secrets); each instance adds its identity from the vault, its ports, the # bootstrap list, and permanent peering with every instance on the OTHER boxes. +# +# Resource manager: System.ConnsInbound 262144, set in +# files/libp2p-resource-limit-overrides.json (kubo dropped +# Swarm.ResourceMgr.Limits; this file in the repo replaced it). Left to kubo it +# is the larger of MaxMemory in MiB (8192) and 2 x ConnMgr.HighWater (64000). +# kubo 0.43.1 leaks inbound TCP/WS connection reservations: in the three days +# after the 2026-10-16 restart each instance gained 5,400-9,000 inbound slots +# more than its live inbound connections (24,619 against 15,632 on sing-1), +# about 2,000-3,200 a day. A leaked slot is accounting only (no socket, no +# memory) and a restart clears them, so the limit is headroom until the next +# restart, not capacity: (262144 - ~16000 live) / 3,200 a day is about 75 days. +# Real connections stay capped by ConnMgr (32000/16000, which also keeps the +# derived 64000 as the fallback if the overrides file is ever missing). +# MaxFileDescriptors 524288 (and IPFS_FD_MAX) because each leaked slot also +# holds an FD slot: 2 x ConnsInbound, as before. These are caps, not use (real +# FDs are about the live TCP connections); the rlimits of chic-1's two +# instances and content together are about 1.2M, under fs.file-max 2097152. # --------------------------------------------------------------------------- - name: "Build the kubo config for {{ bootstrap_namespace }}" vars: @@ -406,6 +423,9 @@ # For probes and CLI calls (kubo's --api-auth). Exposed to the kubo # container as $KUBO_API_AUTH. api-auth: "{{ ('bearer:' ~ bootstrap_instance_api_token) | b64encode }}" + libp2p-resource-limit-overrides.json: >- + {{ lookup('ansible.builtin.file', role_path ~ '/files/libp2p-resource-limit-overrides.json') + | from_json | to_nice_json | b64encode }} no_log: true register: bootstrap_secret # On a first --check the namespace is only simulated and a dry-run secret @@ -475,6 +495,34 @@ success_msg: "{{ bootstrap_namespace }} {{ node.ID }} ({{ node.AgentVersion }}) advertising {{ node.Addresses | length }} addresses" when: not ansible_check_mode +# The overrides file is read only at start; prove the running node uses it, and +# the FD limit from the config. +- name: "Read the resource manager limits {{ bootstrap_namespace }} runs with" + ansible.builtin.command: + argv: [k3s, kubectl, "--namespace={{ bootstrap_namespace }}", exec, deploy/bootstrap, -c, kubo, --, + sh, -c, 'exec ipfs --api={{ bootstrap_api }} --api-auth="$KUBO_API_AUTH" swarm resources --enc=json'] + register: bootstrap_resources + changed_when: false + when: not ansible_check_mode + +- name: Fail unless the resource manager limits match the overrides and the config + vars: + want: "{{ lookup('ansible.builtin.file', role_path ~ '/files/libp2p-resource-limit-overrides.json') | from_json }}" + want_fd: "{{ bootstrap_kubo_config.Swarm.ResourceMgr.MaxFileDescriptors }}" + live: "{{ (bootstrap_resources.stdout | from_json).System }}" + ansible.builtin.assert: + that: + - live.ConnsInbound | int == want.System.ConnsInbound | int + - live.FD | int == want_fd | int + fail_msg: >- + {{ bootstrap_namespace }}: System.ConnsInbound is {{ live.ConnsInbound }}, want {{ want.System.ConnsInbound }}; + System.FD is {{ live.FD }}, want {{ want_fd }} + success_msg: >- + {{ bootstrap_namespace }}: System.ConnsInbound {{ live.ConnsInbound }} ({{ live.ConnsInboundUsage }} in use), + FD {{ live.FD }} ({{ live.FDUsage }} in use) + quiet: false + when: not ansible_check_mode + # From the controller, as a client would: the public WSS port must present a # certificate that verifies for the published name. - name: Check the WSS certificate as a client sees it