diff --git a/components/neutron/values.yaml b/components/neutron/values.yaml index 532c7eeaa..8c2769436 100644 --- a/components/neutron/values.yaml +++ b/components/neutron/values.yaml @@ -67,8 +67,14 @@ conf: DEFAULT: # https://docs.openstack.org/neutron/latest/admin/config-wsgi.html # the api_workers set the number of uWSGI processes as well - api_workers: 4 - rpc_workers: 2 + # Under bursts of concurrent provisioning the default 4 api_workers + # saturated and API requests (e.g. /v2.0/segments) became slow or timed + # out. More API worker processes let neutron-server serve concurrent + # requests. + api_workers: 8 + # more RPC workers to keep up with the OVN/agent RPC traffic generated by + # the same provisioning bursts. + rpc_workers: 4 # We enable the following plugins: # - 'ovn-router' enables OVN to be our L3 router. # - 'trunk' allows for us to create and configure trunk ports to allow @@ -118,8 +124,13 @@ conf: # https://github.com/openstack/neutron/blob/master/neutron/conf/quota.py#L101-L105 quota_rbac_policy: 100 neutron_api_uwsgi: + # deepen the uWSGI listen backlog so bursts of concurrent requests queue + # instead of being refused. With the default backlog, clients saw read + # timeouts against the API during provisioning bursts. + # NOTE: values above the node's net.core.somaxconn are silently capped. uwsgi: start-time: "%t" + listen: 1024 # disable the neutron-ironic-agent from loading a non-existent config pod: @@ -139,8 +150,12 @@ pod: use_fqdn: neutron_agent: false + # raise the default neutron-server replica count. Additional replicas were + # observed to resolve slow/timing-out API responses during heavy concurrent + # provisioning. 8 is a sane baseline for all environments; busier deployments + # can raise this further in their environment-specific values overlay. replicas: - server: 4 + server: 8 lifecycle: disruption_budget: server: @@ -148,17 +163,37 @@ pod: resources: enabled: true server: + # give neutron-server more CPU headroom now that each pod runs 8 + # api_workers + 4 rpc_workers. The previous 200m request under-provisioned + # the pods and there was no CPU limit at all. requests: memory: "2048Mi" - cpu: "200m" + cpu: "500m" limits: memory: "4096Mi" + cpu: "4" + # loosen the neutron-server liveness probe so a slow response during a burst + # of load doesn't trigger a destructive pod restart. A liveness probe should + # detect a dead process, not a busy one. Readiness failing under load is fine + # (pulls the pod from rotation briefly); liveness failing restarts it and makes + # the overload worse. Uses the chart's enabled/params schema. probes: server: server: + liveness: + enabled: true + params: + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 6 readiness: - initialDelaySeconds: 30 - timeoutSeconds: 20 + enabled: true + params: + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 3 # (nicholas.kuechler) updating the jobs list to remove the 'neutron-rabbit-init' job.