feat(neutron): Updates Neutron scaling and performance tuning settings

components/neutron/values.yaml

@@ -67,8 +67,14 @@ conf:
     DEFAULT:
       # https://docs.openstack.org/neutron/latest/admin/config-wsgi.html
       # the api_workers set the number of uWSGI processes as well
-      api_workers: 4
-      rpc_workers: 2
+      # Under bursts of concurrent provisioning the default 4 api_workers
+      # saturated and API requests (e.g. /v2.0/segments) became slow or timed
+      # out. More API worker processes let neutron-server serve concurrent
+      # requests.
+      api_workers: 8
+      # more RPC workers to keep up with the OVN/agent RPC traffic generated by
+      # the same provisioning bursts.
+      rpc_workers: 4
       # We enable the following plugins:
       # - 'ovn-router' enables OVN to be our L3 router.
       # - 'trunk' allows for us to create and configure trunk ports to allow
@@ -118,8 +124,13 @@ conf:
       # https://github.com/openstack/neutron/blob/master/neutron/conf/quota.py#L101-L105
       quota_rbac_policy: 100
   neutron_api_uwsgi:
+    # deepen the uWSGI listen backlog so bursts of concurrent requests queue
+    # instead of being refused. With the default backlog, clients saw read
+    # timeouts against the API during provisioning bursts.
+    # NOTE: values above the node's net.core.somaxconn are silently capped.
     uwsgi:
       start-time: "%t"
+      listen: 1024
 
 # disable the neutron-ironic-agent from loading a non-existent config
 pod:
@@ -139,26 +150,50 @@ pod:
 
   use_fqdn:
     neutron_agent: false
+  # raise the default neutron-server replica count. Additional replicas were
+  # observed to resolve slow/timing-out API responses during heavy concurrent
+  # provisioning. 8 is a sane baseline for all environments; busier deployments
+  # can raise this further in their environment-specific values overlay.
   replicas:
-    server: 4
+    server: 8
   lifecycle:
     disruption_budget:
       server:
         min_available: 2
   resources:
     enabled: true
     server:
+      # give neutron-server more CPU headroom now that each pod runs 8
+      # api_workers + 4 rpc_workers. The previous 200m request under-provisioned
+      # the pods and there was no CPU limit at all.
       requests:
         memory: "2048Mi"
-        cpu: "200m"
+        cpu: "500m"
       limits:
         memory: "4096Mi"
+        cpu: "4"
+  # loosen the neutron-server liveness probe so a slow response during a burst
+  # of load doesn't trigger a destructive pod restart. A liveness probe should
+  # detect a dead process, not a busy one. Readiness failing under load is fine
+  # (pulls the pod from rotation briefly); liveness failing restarts it and makes
+  # the overload worse. Uses the chart's enabled/params schema.
   probes:
     server:
       server:
+        liveness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 10
+            failureThreshold: 6
         readiness:
-          initialDelaySeconds: 30
-          timeoutSeconds: 20
+          enabled: true
+          params:
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 3
 
 
 # (nicholas.kuechler) updating the jobs list to remove the 'neutron-rabbit-init' job.