diff --git a/.konflux/profiles.toml b/.konflux/profiles.toml index 62220c5c8..a7890c9e5 100644 --- a/.konflux/profiles.toml +++ b/.konflux/profiles.toml @@ -1,12 +1,12 @@ [common] python_version = "3.12" platforms = ["x86_64", "aarch64"] -bootstrap_packages = ["maturin", "pip", "uv"] +bootstrap_packages = ["maturin", "uv", "pip"] [profiles.cpu] rhoai_index_url = "https://packages.redhat.com/api/pypi/public-rhai/rhoai/3.5/cpu-ubi9/simple/" output_suffix = "" tekton_files = [ - ".tekton/lightspeed-stack-0-8-pull-request.yaml", - ".tekton/lightspeed-stack-0-8-push.yaml", + ".tekton/lightspeed-stack-0-7-pull-request.yaml", + ".tekton/lightspeed-stack-0-7-push.yaml", ] diff --git a/.konflux/redhat.repo b/.konflux/redhat.repo index a8c6c65c3..334871f20 100644 --- a/.konflux/redhat.repo +++ b/.konflux/redhat.repo @@ -11,6 +11,7 @@ metadata_expire = 86400 enabled_metadata = 0 sslclientkey = $SSL_CLIENT_KEY sslclientcert = $SSL_CLIENT_CERT +excludepkgs=*.i686 [rhel-9-for-$basearch-appstream-eus-rpms__9_DOT_6] name = Red Hat Enterprise Linux 9 for $basearch - AppStream - Extended Update Support (RPMs) @@ -25,6 +26,7 @@ metadata_expire = 86400 enabled_metadata = 0 sslclientkey = $SSL_CLIENT_KEY sslclientcert = $SSL_CLIENT_CERT +excludepkgs=*.i686 [rhel-9-for-$basearch-baseos-eus-rpms__9_DOT_6] name = Red Hat Enterprise Linux 9 for $basearch - BaseOS - Extended Update Support (RPMs) @@ -52,4 +54,4 @@ sslverifystatus = 1 metadata_expire = 86400 enabled_metadata = 0 sslclientkey = $SSL_CLIENT_KEY -sslclientcert = $SSL_CLIENT_CERT \ No newline at end of file +sslclientcert = $SSL_CLIENT_CERT diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml index 54be0647b..e3516e91b 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml @@ -2,17 +2,16 @@ apiVersion: tekton.dev/v1beta1 kind: Pipeline metadata: - name: lightspeed-stack-0-8-integration-tests-pipeline + name: lightspeed-stack-0-7-integration-tests-pipeline spec: description: | This pipeline automates the process of running end-to-end tests for Lightspeed Stack - using a ROSA (Red Hat OpenShift Service on AWS cluster. The pipeline provisions - the ROSA cluster, installs the Lightspeed Stack, runs the tests, collects artifacts, - and finally deprovisions the ROSA cluster. + using an OpenShift CI HyperShift cluster. The pipeline provisions the cluster, + installs the Lightspeed Stack, runs the tests, and collects artifacts. params: - name: SNAPSHOT description: 'The JSON string representing the snapshot of the application under test (includes lightspeed-stack image).' - default: '{"components": [{"name":"lightspeed-stack-0-8", "containerImage": "quay.io/example/lightspeed-stack-0-8:latest"}]}' + default: '{"components": [{"name":"lightspeed-stack-0-7", "containerImage": "quay.io/example/lightspeed-stack-0-7:latest"}]}' type: string - name: llama-stack-image description: 'OGX runs from source on UBI (init container clones repo and installs deps). Parameter name kept for backwards compatibility.' @@ -20,66 +19,49 @@ spec: type: string - name: test-name description: 'The name of the test corresponding to a defined Konflux integration test.' - default: 'lightspeed-stack-0-8-e2e-tests' + default: 'lightspeed-stack-0-7-e2e-tests' - name: namespace description: 'Namespace to run tests in' default: 'lightspeed-stack' + - name: ocp-version + description: 'OpenShift minor version for the ephemeral HyperShift cluster.' + default: '4.19' + type: string + - name: compute-node-type + description: 'AWS instance type for the ephemeral HyperShift compute nodes.' + default: 'm5.xlarge' + type: string + - name: hosted-management-cluster + description: 'Hosted management cluster used to provision the ephemeral cluster.' + default: 'hosted-mgmt2' + type: string tasks: - - name: eaas-provision-space + - name: provision-cluster taskRef: resolver: git params: - name: url - value: https://github.com/konflux-ci/build-definitions.git + value: https://github.com/openshift/konflux-tasks - name: revision value: main - name: pathInRepo - value: task/eaas-provision-space/0.1/eaas-provision-space.yaml + value: tasks/provision-ephemeral-cluster/0.1/provision-ephemeral-cluster.yaml params: - - name: ownerKind - value: PipelineRun - name: ownerName value: $(context.pipelineRun.name) - name: ownerUid value: $(context.pipelineRun.uid) - - name: provision-cluster - runAfter: - - eaas-provision-space - taskSpec: - results: - - name: clusterName - value: "$(steps.create-cluster.results.clusterName)" - steps: - - name: pick-version - ref: - resolver: git - params: - - name: url - value: https://github.com/konflux-ci/build-definitions.git - - name: revision - value: main - - name: pathInRepo - value: stepactions/eaas-get-latest-openshift-version-by-prefix/0.1/eaas-get-latest-openshift-version-by-prefix.yaml - params: - - name: prefix - value: "4.19." - - name: create-cluster - ref: - resolver: git - params: - - name: url - value: https://github.com/konflux-ci/build-definitions.git - - name: revision - value: main - - name: pathInRepo - value: stepactions/eaas-create-ephemeral-cluster-hypershift-aws/0.1/eaas-create-ephemeral-cluster-hypershift-aws.yaml - params: - - name: eaasSpaceSecretRef - value: $(tasks.eaas-provision-space.results.secretRef) - - name: version - value: "$(steps.pick-version.results.version)" - - name: instanceType - value: "m5.large" + - name: workflow + value: hypershift-hostedcluster-workflow + - name: clusterProfile + value: aws-konflux-prod + - name: timeout + value: 1h + - name: releases + value: >- + {"latest":{"release":{"channel":"stable","version":"$(params.ocp-version)","architecture":"multi"}}} + - name: env + value: '{"COMPUTE_NODE_TYPE":"$(params.compute-node-type)","HOSTED_MANAGEMENT_CLUSTER":"$(params.hosted-management-cluster)"}' - name: get-stack-images description: Extract lightspeed-stack image and commit from SNAPSHOT (OGX runs from source in-pod) runAfter: @@ -117,8 +99,8 @@ spec: description: "commit sha to be used to store artifacts" script: | dnf -y install jq - echo -n "$(jq -r --arg n "lightspeed-stack-0-8" '.components[] | select(.name == $n) | .containerImage // ""' <<< "$SNAPSHOT")" > $(step.results.lightspeed-stack-image.path) - echo -n "$(jq -r --arg n "lightspeed-stack-0-8" '.components[] | select(.name == $n) | .source.git.revision // "latest"' <<< "$SNAPSHOT")" > $(step.results.commit.path) + echo -n "$(jq -r --arg n "lightspeed-stack-0-7" '.components[] | select(.name == $n) | .containerImage // ""' <<< "$SNAPSHOT")" > $(step.results.lightspeed-stack-image.path) + echo -n "$(jq -r --arg n "lightspeed-stack-0-7" '.components[] | select(.name == $n) | .source.git.revision // "latest"' <<< "$SNAPSHOT")" > $(step.results.commit.path) - name: echo-integration-params description: Echo all params passed to lightspeed-stack-integration-tests for verification before the test runs. runAfter: @@ -180,10 +162,10 @@ spec: value: $(tasks.get-stack-images.results.commit) - name: namespace value: "$(params.namespace)" - - name: spaceRequestSecretName - value: $(tasks.eaas-provision-space.results.secretRef) - - name: clusterName - value: $(tasks.provision-cluster.results.clusterName) + - name: clusterCredentialsSecretRef + value: $(tasks.provision-cluster.results.secretRef) + - name: ocp-version + value: $(params.ocp-version) runAfter: - echo-integration-params taskSpec: @@ -194,9 +176,9 @@ spec: - name: commit - name: namespace type: string - - name: spaceRequestSecretName + - name: clusterCredentialsSecretRef type: string - - name: clusterName + - name: ocp-version type: string results: - name: TEST_OUTPUT @@ -217,25 +199,9 @@ spec: secret: secretName: redhat-registry-io-pull-secret - name: credentials - emptyDir: {} + secret: + secretName: $(params.clusterCredentialsSecretRef) steps: - - name: get-kubeconfig - ref: - resolver: git - params: - - name: url - value: https://github.com/konflux-ci/build-definitions.git - - name: revision - value: main - - name: pathInRepo - value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml - params: - - name: eaasSpaceSecretRef - value: $(params.spaceRequestSecretName) - - name: clusterName - value: "$(params.clusterName)" - - name: credentials - value: credentials - name: run-e2e-tests #onError: continue # Do not use onError: continue here unless a later step like @@ -262,7 +228,7 @@ spec: mountPath: /credentials env: - name: KUBECONFIG - value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + value: "/credentials/kubeconfig" - name: ARTIFACT_DIR value: "/workspace/artifacts" - name: SUITE_ID @@ -288,12 +254,12 @@ spec: echo "[e2e] 2/8 Installing deps (git, tar, jq, curl-minimal, python3, gettext for envsubst)..." microdnf -y install git tar jq curl-minimal python3 gettext echo "[e2e] 3/8 Downloading oc client..." - curl -sL -o oc.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest-4.19/openshift-client-linux-amd64-rhel9.tar.gz + curl -sL -o oc.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest-$(params.ocp-version)/openshift-client-linux-amd64-rhel9.tar.gz tar -xzf oc.tar.gz && chmod +x kubectl oc && mv oc kubectl /usr/local/bin/ echo "[e2e] 4/8 SNAPSHOT (length ${#SNAPSHOT} chars; clone URL fixed — SNAPSHOT is main/upstream, not fork)..." # Fixed fork + branch: Konflux SNAPSHOT points at main repo/rev, not the PR/fork under test. - REPO_URL=$(jq -r '.components[] | select(.name == "lightspeed-stack-0-8") | .source.git.url // "https://github.com/lightspeed-core/lightspeed-stack.git"' <<< "$SNAPSHOT") - REPO_REV=$(jq -r '.components[] | select(.name == "lightspeed-stack-0-8") | .source.git.revision // "main"' <<< "$SNAPSHOT") + REPO_URL=$(jq -r '.components[] | select(.name == "lightspeed-stack-0-7") | .source.git.url // "https://github.com/lightspeed-core/lightspeed-stack.git"' <<< "$SNAPSHOT") + REPO_REV=$(jq -r '.components[] | select(.name == "lightspeed-stack-0-7") | .source.git.revision // "main"' <<< "$SNAPSHOT") echo "[e2e] 5/8 Clone $REPO_URL @ $REPO_REV" git clone -q "$REPO_URL" /workspace/lightspeed-stack cd /workspace/lightspeed-stack && git fetch origin "$REPO_REV" && git checkout -q "$REPO_REV" @@ -319,7 +285,7 @@ spec: # - name: credentials # value: "credentials" # - name: kubeconfig - # value: "$(steps.get-kubeconfig.results.kubeconfig)" + # value: "kubeconfig" # - name: artifact-dir # value: "/workspace/konflux-artifacts" # # validate that the cluster resources are available in another tekton step @@ -359,4 +325,4 @@ spec: # - name: revision # value: main # - name: pathInRepo - # value: stepactions/fail-if-any-step-failed/0.1/fail-if-any-step-failed.yaml \ No newline at end of file + # value: stepactions/fail-if-any-step-failed/0.1/fail-if-any-step-failed.yaml diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml index 5b24d6d27..986bcb65a 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml @@ -2,20 +2,20 @@ apiVersion: tekton.dev/v1beta1 kind: Pipeline metadata: - name: lightspeed-stack-0-8-rhelai-tests-pipeline + name: lightspeed-stack-0-7-rhelai-tests-pipeline spec: description: | This pipeline provisions a RHEL AI instance on AWS (vLLM), an ephemeral - OpenShift cluster, deploys lightspeed-stack configured to use the RHEL AI - vLLM as inference provider, and runs the full e2e test suite. + OpenShift CI HyperShift cluster, deploys lightspeed-stack configured to use + the RHEL AI vLLM as inference provider, and runs the full e2e test suite. params: - name: SNAPSHOT description: 'The JSON string representing the snapshot of the application under test.' - default: '{"components": [{"name":"lightspeed-stack-0-8", "containerImage": "quay.io/example/lightspeed-stack-0-8:latest"}]}' + default: '{"components": [{"name":"lightspeed-stack-0-7", "containerImage": "quay.io/example/lightspeed-stack-0-7:latest"}]}' type: string - name: test-name description: 'The name of the test corresponding to a defined Konflux integration test.' - default: 'lightspeed-stack-0-8-rhelai-tests' + default: 'lightspeed-stack-0-7-rhelai-tests' - name: rhelai-version description: 'RHEL AI version to provision.' default: '3.4.0' @@ -39,6 +39,18 @@ spec: description: 'Enable debug output.' default: 'false' type: string + - name: ocp-version + description: 'OpenShift minor version for the ephemeral HyperShift cluster.' + default: '4.19' + type: string + - name: compute-node-type + description: 'AWS instance type for the ephemeral HyperShift compute nodes.' + default: 'm5.xlarge' + type: string + - name: hosted-management-cluster + description: 'Hosted management cluster used to provision the ephemeral cluster.' + default: 'hosted-mgmt2' + type: string tasks: # ── RHEL AI provisioning (MAPT) ── - name: provision-rhelai @@ -199,63 +211,33 @@ spec: - name: spot value: "$(params.spot)" - # ── OpenShift cluster provisioning (same as existing e2e pipeline) ── - - name: eaas-provision-space + # ── OpenShift CI HyperShift cluster provisioning ── + - name: provision-cluster taskRef: resolver: git params: - name: url - value: https://github.com/konflux-ci/build-definitions.git + value: https://github.com/openshift/konflux-tasks - name: revision value: main - name: pathInRepo - value: task/eaas-provision-space/0.1/eaas-provision-space.yaml + value: tasks/provision-ephemeral-cluster/0.1/provision-ephemeral-cluster.yaml params: - - name: ownerKind - value: PipelineRun - name: ownerName value: $(context.pipelineRun.name) - name: ownerUid value: $(context.pipelineRun.uid) - - - name: provision-cluster - runAfter: - - eaas-provision-space - taskSpec: - results: - - name: clusterName - value: "$(steps.create-cluster.results.clusterName)" - steps: - - name: pick-version - ref: - resolver: git - params: - - name: url - value: https://github.com/konflux-ci/build-definitions.git - - name: revision - value: main - - name: pathInRepo - value: stepactions/eaas-get-latest-openshift-version-by-prefix/0.1/eaas-get-latest-openshift-version-by-prefix.yaml - params: - - name: prefix - value: "4.19." - - name: create-cluster - ref: - resolver: git - params: - - name: url - value: https://github.com/konflux-ci/build-definitions.git - - name: revision - value: main - - name: pathInRepo - value: stepactions/eaas-create-ephemeral-cluster-hypershift-aws/0.1/eaas-create-ephemeral-cluster-hypershift-aws.yaml - params: - - name: eaasSpaceSecretRef - value: $(tasks.eaas-provision-space.results.secretRef) - - name: version - value: "$(steps.pick-version.results.version)" - - name: instanceType - value: "m5.large" + - name: workflow + value: hypershift-hostedcluster-workflow + - name: clusterProfile + value: aws-konflux-prod + - name: timeout + value: 1h + - name: releases + value: >- + {"latest":{"release":{"channel":"stable","version":"$(params.ocp-version)","architecture":"multi"}}} + - name: env + value: '{"COMPUTE_NODE_TYPE":"$(params.compute-node-type)","HOSTED_MANAGEMENT_CLUSTER":"$(params.hosted-management-cluster)"}' - name: get-stack-images description: Extract lightspeed-stack image and commit from SNAPSHOT @@ -285,8 +267,8 @@ spec: type: string script: | dnf -y install jq - echo -n "$(jq -r --arg n "lightspeed-stack-0-8" '.components[] | select(.name == $n) | .containerImage // ""' <<< "$SNAPSHOT")" > $(step.results.lightspeed-stack-image.path) - echo -n "$(jq -r --arg n "lightspeed-stack-0-8" '.components[] | select(.name == $n) | .source.git.revision // "latest"' <<< "$SNAPSHOT")" > $(step.results.commit.path) + echo -n "$(jq -r --arg n "lightspeed-stack-0-7" '.components[] | select(.name == $n) | .containerImage // ""' <<< "$SNAPSHOT")" > $(step.results.lightspeed-stack-image.path) + echo -n "$(jq -r --arg n "lightspeed-stack-0-7" '.components[] | select(.name == $n) | .source.git.revision // "latest"' <<< "$SNAPSHOT")" > $(step.results.commit.path) # ── Full E2E tests (runs after both RHEL AI and OpenShift are ready) ── - name: rhelai-e2e-tests @@ -304,10 +286,10 @@ spec: value: $(tasks.get-stack-images.results.commit) - name: namespace value: "$(params.namespace)" - - name: spaceRequestSecretName - value: $(tasks.eaas-provision-space.results.secretRef) - - name: clusterName - value: $(tasks.provision-cluster.results.clusterName) + - name: clusterCredentialsSecretRef + value: $(tasks.provision-cluster.results.secretRef) + - name: ocp-version + value: $(params.ocp-version) - name: vllm-host value: "$(tasks.provision-rhelai.results.host)" - name: vllm-api-key @@ -321,9 +303,9 @@ spec: - name: commit - name: namespace type: string - - name: spaceRequestSecretName + - name: clusterCredentialsSecretRef type: string - - name: clusterName + - name: ocp-version type: string - name: vllm-host type: string @@ -342,25 +324,9 @@ spec: secret: secretName: quay-aipcc-password - name: credentials - emptyDir: {} + secret: + secretName: $(params.clusterCredentialsSecretRef) steps: - - name: get-kubeconfig - ref: - resolver: git - params: - - name: url - value: https://github.com/konflux-ci/build-definitions.git - - name: revision - value: main - - name: pathInRepo - value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml - params: - - name: eaasSpaceSecretRef - value: $(params.spaceRequestSecretName) - - name: clusterName - value: "$(params.clusterName)" - - name: credentials - value: credentials - name: run-e2e-tests resources: requests: @@ -379,7 +345,7 @@ spec: mountPath: /credentials env: - name: KUBECONFIG - value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + value: "/credentials/kubeconfig" - name: ARTIFACT_DIR value: "/workspace/artifacts" - name: KONFLUX_BOOL @@ -407,11 +373,11 @@ spec: echo "[e2e] 2/8 Installing deps..." microdnf -y install git tar jq curl-minimal python3 gettext echo "[e2e] 3/8 Downloading oc client..." - curl -sL -o oc.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest-4.19/openshift-client-linux-amd64-rhel9.tar.gz + curl -sL -o oc.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest-$(params.ocp-version)/openshift-client-linux-amd64-rhel9.tar.gz tar -xzf oc.tar.gz && chmod +x kubectl oc && mv oc kubectl /usr/local/bin/ echo "[e2e] 4/8 VLLM_URL=$VLLM_URL" - REPO_URL=$(jq -r '.components[] | select(.name == "lightspeed-stack-0-8") | .source.git.url // "https://github.com/lightspeed-core/lightspeed-stack.git"' <<< "$SNAPSHOT") - REPO_REV=$(jq -r '.components[] | select(.name == "lightspeed-stack-0-8") | .source.git.revision // "main"' <<< "$SNAPSHOT") + REPO_URL=$(jq -r '.components[] | select(.name == "lightspeed-stack-0-7") | .source.git.url // "https://github.com/lightspeed-core/lightspeed-stack.git"' <<< "$SNAPSHOT") + REPO_REV=$(jq -r '.components[] | select(.name == "lightspeed-stack-0-7") | .source.git.revision // "main"' <<< "$SNAPSHOT") echo "[e2e] 5/8 Clone $REPO_URL @ $REPO_REV" git clone -q "$REPO_URL" /workspace/lightspeed-stack cd /workspace/lightspeed-stack && git fetch origin "$REPO_REV" && git checkout -q "$REPO_REV" diff --git a/.tekton/lightspeed-stack-0-8-pull-request.yaml b/.tekton/lightspeed-stack-0-7-pull-request.yaml similarity index 94% rename from .tekton/lightspeed-stack-0-8-pull-request.yaml rename to .tekton/lightspeed-stack-0-7-pull-request.yaml index caeae47b2..493862703 100644 --- a/.tekton/lightspeed-stack-0-8-pull-request.yaml +++ b/.tekton/lightspeed-stack-0-7-pull-request.yaml @@ -9,12 +9,12 @@ metadata: pipelinesascode.tekton.dev/cancel-in-progress: "true" pipelinesascode.tekton.dev/max-keep-runs: "3" pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch - == "main" + == "release/0.7" labels: - appstudio.openshift.io/application: lightspeed-core-0-8 - appstudio.openshift.io/component: lightspeed-stack-0-8 + appstudio.openshift.io/application: lightspeed-core-0-7 + appstudio.openshift.io/component: lightspeed-stack-0-7 pipelines.appstudio.openshift.io/type: build - name: lightspeed-stack-0-8-on-pull-request + name: lightspeed-stack-0-7-on-pull-request namespace: lightspeed-core-tenant spec: params: @@ -23,7 +23,7 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack-0-8:on-pr-{{revision}} + value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack-0-7:on-pr-{{revision}} - name: image-expires-after value: 5d - name: build-platforms @@ -269,8 +269,6 @@ spec: - name: BUILD_ARGS value: - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - name: PRIVILEGED_NESTED value: $(params.privileged-nested) - name: SOURCE_URL @@ -293,6 +291,9 @@ spec: value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: IMAGE_APPEND_PLATFORM value: "true" + - name: BUILD_ARGS_FILES + value: + - $(params.build-args-file) runAfter: - prefetch-dependencies taskRef: @@ -300,7 +301,7 @@ spec: - name: name value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.12.3@sha256:88b971c800b3e71903ddad83da884d092e9beda8b51deaf980f6525e45ed043f + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.13.0@sha256:04dbc0e479ebc2d7c94a4e1cde84aaf3f6ab84b3e1bbae012a4554219a297618 - name: kind value: task resolver: bundles @@ -365,7 +366,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:0ccc688a77e9b7b0b8973c132a1e840844137e77f887be4a0bec8893b0776872 + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:81b7cad8d5fd0148b157d622f4064f57d38aa33c142275050c8a4727d1e2144c - name: kind value: task resolver: bundles @@ -392,7 +393,7 @@ spec: - name: name value: roxctl-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:4d48db2ada6584bc7343ee11adb04d86f1ca9a2849dc718ccda861e092fcf41b + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:e398b9ddfc422639c2e245285577b01b8c25cc141a2563e1d85449d96b619752 - name: kind value: task resolver: bundles @@ -417,7 +418,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:fa53ef450ba538addafee1211ff6ac15b55865ce652fec15ac99b554680cb642 + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:439014132f4521ea3221ea29b1fd83e48f9728c071db6edaf6b870550e5d89a6 - name: kind value: task resolver: bundles @@ -434,8 +435,6 @@ spec: value: $(tasks.build-image-index.results.IMAGE_URL) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: ARGS value: --project-name=lightspeed-stack --report --org=dca2ca89-7e51-4a3a-b7a5-6ad5633057b8 - name: TARGET_DIRS @@ -447,7 +446,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5@sha256:17b9587eba53fabaab35061275306bfe1f99661d55e14e22f62321fa7cb5ff3e + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5.1@sha256:d01cd72c9884bd6db08dcc558e6cb6a5f408bc905a3aaa7ece6cf080c933108f - name: kind value: task resolver: bundles @@ -474,7 +473,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.4@sha256:3203cd3bde82d9e4d1beb5f1366942346800cff89e392c920733eb0ea78d731b + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.4@sha256:6fe7e04cef9d999aeb62a29839df82a24fd4841c93998bb0c4ef3cd9a69288c2 - name: kind value: task resolver: bundles @@ -559,8 +558,6 @@ spec: value: $(tasks.build-image-index.results.IMAGE_URL) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: TARGET_DIRS value: $(params.sast-target-dirs) runAfter: @@ -570,7 +567,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d00c884c78489a52dc638e6ced0a01429670d0754f5b1305ad576709cf660312 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1.1@sha256:a3bcbf008dc76df0d7d91d17b4e2cf925395fe9470361283d25bba73958c7281 - name: kind value: task resolver: bundles @@ -589,8 +586,6 @@ spec: value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -598,7 +593,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:15d654c9576b9c4ee1f78b09fce541acf9afe33f1c2363fd6219a79dbf62c95d + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4.1@sha256:ec176a34ddf4392e06d6904b4b2b2a58e34db0487114e2a476796b1310760fad - name: kind value: task resolver: bundles @@ -620,7 +615,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.1@sha256:c177fc2b2482b1299e713c51bf258eba4972c016ef260d9f45335c2550dc7b2b + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.2@sha256:a1d5e6d0912de4e39a19b4aa5587526cb8b20260ac32d00ac5c8c4581e3c2ea1 - name: kind value: task resolver: bundles @@ -660,7 +655,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:4b1856e41d087e53af26562fb16a490579777d9ff05af33011ce212d477c7cd5 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:f72e0baac32f1fc70e441cb2e84b7d1d54857bf520e0d7db68ae35174fb46709 - name: kind value: task resolver: bundles @@ -678,7 +673,7 @@ spec: pipeline: 4h tasks: 4h taskRunTemplate: - serviceAccountName: build-pipeline-lightspeed-stack-0-8 + serviceAccountName: build-pipeline-lightspeed-stack-0-7 workspaces: - name: git-auth secret: diff --git a/.tekton/lightspeed-stack-0-8-push.yaml b/.tekton/lightspeed-stack-0-7-push.yaml similarity index 94% rename from .tekton/lightspeed-stack-0-8-push.yaml rename to .tekton/lightspeed-stack-0-7-push.yaml index 0d5eb6b53..b4faeab78 100644 --- a/.tekton/lightspeed-stack-0-8-push.yaml +++ b/.tekton/lightspeed-stack-0-7-push.yaml @@ -12,12 +12,12 @@ metadata: pipelinesascode.tekton.dev/cancel-in-progress: "false" pipelinesascode.tekton.dev/max-keep-runs: "3" pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch - == "main" + == "release/0.7" labels: - appstudio.openshift.io/application: lightspeed-core-0-8 - appstudio.openshift.io/component: lightspeed-stack-0-8 + appstudio.openshift.io/application: lightspeed-core-0-7 + appstudio.openshift.io/component: lightspeed-stack-0-7 pipelines.appstudio.openshift.io/type: build - name: lightspeed-stack-0-8-on-push + name: lightspeed-stack-0-7-on-push namespace: lightspeed-core-tenant spec: params: @@ -26,7 +26,7 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack-0-8:{{revision}} + value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack-0-7:{{revision}} - name: build-platforms value: - linux/x86_64 @@ -270,8 +270,6 @@ spec: - name: BUILD_ARGS value: - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - name: PRIVILEGED_NESTED value: $(params.privileged-nested) - name: SOURCE_URL @@ -294,6 +292,9 @@ spec: value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: IMAGE_APPEND_PLATFORM value: "true" + - name: BUILD_ARGS_FILES + value: + - $(params.build-args-file) runAfter: - prefetch-dependencies taskRef: @@ -301,7 +302,7 @@ spec: - name: name value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.12.3@sha256:88b971c800b3e71903ddad83da884d092e9beda8b51deaf980f6525e45ed043f + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.13.0@sha256:04dbc0e479ebc2d7c94a4e1cde84aaf3f6ab84b3e1bbae012a4554219a297618 - name: kind value: task resolver: bundles @@ -366,7 +367,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:0ccc688a77e9b7b0b8973c132a1e840844137e77f887be4a0bec8893b0776872 + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:81b7cad8d5fd0148b157d622f4064f57d38aa33c142275050c8a4727d1e2144c - name: kind value: task resolver: bundles @@ -393,7 +394,7 @@ spec: - name: name value: roxctl-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:4d48db2ada6584bc7343ee11adb04d86f1ca9a2849dc718ccda861e092fcf41b + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:e398b9ddfc422639c2e245285577b01b8c25cc141a2563e1d85449d96b619752 - name: kind value: task resolver: bundles @@ -418,7 +419,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:fa53ef450ba538addafee1211ff6ac15b55865ce652fec15ac99b554680cb642 + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:439014132f4521ea3221ea29b1fd83e48f9728c071db6edaf6b870550e5d89a6 - name: kind value: task resolver: bundles @@ -435,8 +436,6 @@ spec: value: $(tasks.build-image-index.results.IMAGE_URL) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: ARGS value: --project-name=lightspeed-stack --report --org=dca2ca89-7e51-4a3a-b7a5-6ad5633057b8 - name: TARGET_DIRS @@ -448,7 +447,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5@sha256:17b9587eba53fabaab35061275306bfe1f99661d55e14e22f62321fa7cb5ff3e + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5.1@sha256:d01cd72c9884bd6db08dcc558e6cb6a5f408bc905a3aaa7ece6cf080c933108f - name: kind value: task resolver: bundles @@ -475,7 +474,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.4@sha256:3203cd3bde82d9e4d1beb5f1366942346800cff89e392c920733eb0ea78d731b + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.4@sha256:6fe7e04cef9d999aeb62a29839df82a24fd4841c93998bb0c4ef3cd9a69288c2 - name: kind value: task resolver: bundles @@ -560,8 +559,6 @@ spec: value: $(tasks.build-image-index.results.IMAGE_URL) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: TARGET_DIRS value: $(params.sast-target-dirs) runAfter: @@ -571,7 +568,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d00c884c78489a52dc638e6ced0a01429670d0754f5b1305ad576709cf660312 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1.1@sha256:a3bcbf008dc76df0d7d91d17b4e2cf925395fe9470361283d25bba73958c7281 - name: kind value: task resolver: bundles @@ -590,8 +587,6 @@ spec: value: $(params.sast-target-dirs) - name: SOURCE_ARTIFACT value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: @@ -599,7 +594,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:15d654c9576b9c4ee1f78b09fce541acf9afe33f1c2363fd6219a79dbf62c95d + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4.1@sha256:ec176a34ddf4392e06d6904b4b2b2a58e34db0487114e2a476796b1310760fad - name: kind value: task resolver: bundles @@ -624,7 +619,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.1@sha256:c177fc2b2482b1299e713c51bf258eba4972c016ef260d9f45335c2550dc7b2b + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.2@sha256:a1d5e6d0912de4e39a19b4aa5587526cb8b20260ac32d00ac5c8c4581e3c2ea1 - name: kind value: task resolver: bundles @@ -664,7 +659,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:4b1856e41d087e53af26562fb16a490579777d9ff05af33011ce212d477c7cd5 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:f72e0baac32f1fc70e441cb2e84b7d1d54857bf520e0d7db68ae35174fb46709 - name: kind value: task resolver: bundles @@ -682,7 +677,7 @@ spec: pipeline: 4h tasks: 4h taskRunTemplate: - serviceAccountName: build-pipeline-lightspeed-stack-0-8 + serviceAccountName: build-pipeline-lightspeed-stack-0-7 workspaces: - name: git-auth secret: diff --git a/deploy/lightspeed-stack/Containerfile b/deploy/lightspeed-stack/Containerfile index 6160daf5e..bfd70e178 100644 --- a/deploy/lightspeed-stack/Containerfile +++ b/deploy/lightspeed-stack/Containerfile @@ -94,8 +94,6 @@ COPY --from=builder /app-root/LICENSE /licenses/ USER root # Additional tools for derived images -# Note: --allowerasing is only supported by dnf, not microdnf (used by the -# *-minimal base images), so it is only appended when RUNTIME_DNF_COMMAND=dnf. RUN ${RUNTIME_DNF_COMMAND} install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs jq patch && \ ${RUNTIME_DNF_COMMAND} update -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs @@ -128,7 +126,7 @@ ENTRYPOINT ["/app-root/entrypoint.sh"] LABEL vendor="Red Hat, Inc." \ name="lightspeed-core/lightspeed-stack-rhel9" \ com.redhat.component="lightspeed-core/lightspeed-stack" \ - cpe="cpe:/a:redhat:lightspeed_core:0.8::el9" \ + cpe="cpe:/a:redhat:lightspeed_core:0.7::el9" \ io.k8s.display-name="Lightspeed Stack" \ summary="A service that provides a REST API for the Lightspeed Core Stack." \ description="Lightspeed Core Stack (LCS) is an AI-powered assistant that provides answers to product questions using backend LLM services, agents, and RAG databases." \ diff --git a/docs/design/conversation-compaction/conversation-compaction.md b/docs/design/conversation-compaction/conversation-compaction.md index 9abe1dbc0..6c70380be 100644 --- a/docs/design/conversation-compaction/conversation-compaction.md +++ b/docs/design/conversation-compaction/conversation-compaction.md @@ -61,7 +61,7 @@ R9 Compaction configuration must be admin-configurable via YAML: threshold ratio, fixed token floor, and buffer zone size. R10 -After compaction, lightspeed-stack builds the LLM input explicitly — the summaries plus the recent verbatim turns plus the new query — and stops passing the OGX `conversation` parameter for that request, because OGX always reloads the full message history when the `conversation` parameter is set (verified empirically on OGX 0.6.0; see the Changelog and the spike doc). The summary is still written into the conversation as a marked item so it appears in the OGX Conversations API, but the marker is lightspeed-stack's own boundary bookkeeping, not an OGX selection mechanism. The `conversation_id` is preserved across the whole conversation, and the full history (including pre-compaction turns) remains in the conversation's items for UI/audit. Because the `conversation` parameter is no longer sent in compacted mode, lightspeed-stack appends each completed turn to the conversation itself. +After compaction, lightspeed-stack builds the LLM input explicitly — the summaries plus the recent verbatim turns plus the new query — and stops passing the OGX `conversation` parameter for that request, because OGX always reloads the full message history when the `conversation` parameter is set (verified empirically on OGX 0.6.0; see the Changelog and the spike doc). The summary is still written into the conversation as a marked item so it appears in the Conversations API, but the marker is lightspeed-stack's own boundary bookkeeping, not an OGX selection mechanism. The `conversation_id` is preserved across the whole conversation, and the full history (including pre-compaction turns) remains in the conversation's items for UI/audit. Because the `conversation` parameter is no longer sent in compacted mode, lightspeed-stack appends each completed turn to the conversation itself. This applies to every endpoint that builds context from a growing conversation and calls the Responses API: `/v1/query`, `/v1/streaming_query`, the A2A executor, and `/v1/responses`. (The `/v1/rlsapi` inference path is stateless — no stored conversation — and is therefore out of scope.) @@ -229,11 +229,11 @@ As built, the cache is the **preferred source of truth** for summary text at run ## Changed request flow after compaction -After compaction, lightspeed-stack writes the summary as a marked conversation item (a message whose text begins with a recognizable sentinel) so it appears in the OGX Conversations API and serves as lightspeed-stack's own boundary marker. +After compaction, lightspeed-stack writes the summary as a marked conversation item (a message whose text begins with a recognizable sentinel) so it appears in the Conversations API and serves as lightspeed-stack's own boundary marker. When building context for a compacted conversation, lightspeed-stack fetches the conversation items, reads the active summaries from the summary cache (LCORE-1571) — falling back to the marker texts when no persisting cache is configured — takes the items after the last marker as the recent verbatim buffer, and sends `[summaries] + [recent items] + [new query]` as **explicit input**, **without** the `conversation` parameter. This is necessary because OGX reloads the *full* stored message history whenever the `conversation` parameter is set — there is no marker-based selection hook (verified empirically; see the Changelog). Each completed turn is then appended back to the conversation items by lightspeed-stack, since OGX no longer auto-stores it. -This preserves a single continuous conversation identity. The `conversation_id` never changes and the user sees one conversation in the UI. OGX stores the full history, including the summary marker items, and returns it through its Conversations API. lightspeed-stack's own `GET /v1/conversations/{conversation_id}` leaves the marker items out of the chat history it returns (LCORE-3909): they are stored as user messages, but the user never sent them. Stored conversations are not migrated: the markers must stay in storage as the fallback source of truth, so they are filtered when the conversation is read. +This preserves a single continuous conversation identity. The `conversation_id` never changes, the user sees one conversation in the UI, and the Conversations API returns the full history including the summary marker items. ## API response changes @@ -323,7 +323,6 @@ Add `compaction` field to the root `Configuration` class. | `src/models/api/responses/successful/query.py` | `context_status` on `QueryResponse` (non-streaming `/v1/query`) — LCORE-1573 | | `src/models/common/agents/stream_payloads.py` | `context_status` on `EndEventData`, the streaming SSE `end` event payload (`StreamingQueryResponse` is docs-only and intentionally skipped) — LCORE-1573 | | `src/cache/` (all backends) | `ConversationSummary` storage — LCORE-1571 | -| `src/app/endpoints/conversations_v1.py` | Pass the stored items through `exclude_marker_items()` before building the chat history — LCORE-3909 | ## How compaction is invoked @@ -436,16 +435,6 @@ mid-conversation on an already-compacted conversation reverts it to full-history replay (unsupported — see Configuration). The `CompactionResult` mode flag was renamed `summarized` → `compacted` for clarity. -**2026-09-28 — Markers are left out of the conversation history (LCORE-3909).** -`GET /v1/conversations/{conversation_id}` filters the marker items when the -conversation is read. A marker is a user message that starts with the sentinel, -with or without the covered-item count. The markers stay in storage as the -fallback source of truth, and stored conversations are not migrated. -`GET /v2/conversations/{conversation_id}` reads the conversation cache, which -never holds a marker, and is unchanged. Earlier revisions of this document said -that the Conversations API returns the marker items; that holds for the OGX -Conversations API only. - # Appendix A: PoC Evidence A proof-of-concept was built and tested. diff --git a/docs/design/ogx-config-merge/ogx-config-merge.md b/docs/design/ogx-config-merge/ogx-config-merge.md index 2521970c9..b0e7b2d8c 100644 --- a/docs/design/ogx-config-merge/ogx-config-merge.md +++ b/docs/design/ogx-config-merge/ogx-config-merge.md @@ -41,8 +41,7 @@ Key shape: `llama_stack` YAML-section alias is still accepted) + external `run.yaml`) is preserved during a deprecation window; mutually exclusive with the unified *synthesis inputs* (a non-empty - `inference.providers`, a non-empty `vector_store.providers`, or an - `ogx.config` block). + `inference.providers` or a `ogx.config` block). ## Why @@ -120,8 +119,7 @@ detail that LCORE owns, not an operator-facing artifact. location for debugging. - **R11:** Shape detection determines mode. Unified mode is signalled by the presence of any *synthesis input* — a non-empty top-level - `inference.providers`, a non-empty top-level `vector_store.providers`, - or an `ogx.config` block; legacy mode by + `inference.providers` or a `ogx.config` block; legacy mode by `library_client_config_path`. An optional `config_format_version` field is accepted but must agree with the detected shape when present. See the "Mode detection" table under Architecture for the full matrix. @@ -180,10 +178,10 @@ lightspeed-stack.yaml (unified mode) └────────────┬───────────────┘ and `library_client_config_path`. │ Configuration (typed) ▼ - ┌────────────────────────────┐ Baseline selection (profile / default / - │ Synthesizer │ byo-llm / empty) + high-level sections - │ synthesize_configuration │ + native_override deep-merge, then - │ (ogx_config…) │ enrichment (BYOK RAG, Solr/OKP, Azure). + ┌────────────────────────────┐ Baseline selection (profile / + │ Synthesizer │ default / empty) + enrichment + │ synthesize_configuration │ (BYOK RAG, Solr/OKP) + high-level + │ (ogx_config…) │ sections + native_override deep-merge. └────────────┬───────────────┘ │ synthesized run.yaml (dict) ▼ @@ -199,23 +197,23 @@ lightspeed-stack.yaml (unified mode) ### Trigger mechanism At LCORE startup (library mode): if any synthesis input is present (a -non-empty top-level `inference.providers`, a non-empty top-level -`vector_store.providers`, or an `ogx.config` block), the synthesizer produces a `run.yaml` dict, writes it to disk, +non-empty top-level `inference.providers`, or a `ogx.config` +block), the synthesizer produces a `run.yaml` dict, writes it to disk, and passes the path to the library client. At OGX container startup (server mode): the container's entrypoint script invokes `python3 /opt/app-root/ogx_configuration.py -c --i -o /tmp/generated-run.yaml` (`-i` is used in legacy mode only). The Python CLI auto-detects unified vs legacy +-o /opt/app-root/run.yaml`. The Python CLI auto-detects unified vs legacy by the same synthesis-input check; in unified mode it synthesizes and writes the output; in legacy mode it performs in-place enrichment as before. ### Mode detection -*Synthesis inputs* are the top-level high-level sections (a non-empty -`inference.providers` or a non-empty `vector_store.providers`; future -`rag`, …) and the `ogx.config` block. The loaded `lightspeed-stack.yaml` maps to a mode as follows: +*Synthesis inputs* are the top-level high-level sections (v1: a non-empty +`inference.providers`; future `rag`, …) and the `ogx.config` +block. The loaded `lightspeed-stack.yaml` maps to a mode as follows: | Shape | Mode | |---|---| @@ -475,16 +473,15 @@ not have had a full release with a working migration path. Releases: is `byo-llm`, strip the built-in conditional OpenAI inference row. `empty` and `profile:` are unchanged. 3. Run `dedupe_providers_vector_io` on the baseline. -4. If top-level `inference.providers` is non-empty → +4. Apply existing enrichment: `enrich_byok_rag`, `enrich_solr` (Azure + Entra ID intentionally stays separate because it's a `.env` + side-effect, not an `ogx_config` mutation). +5. If top-level `inference.providers` is non-empty → `apply_high_level_inference(ogx_config, lcs_config["inference"])`. -5. Unless the baseline is `empty` → `ensure_mcp_tool_runtime(ogx_config)`. 6. If `unified` and `unified.native_override` non-empty → `deep_merge_list_replace(ogx_config, native_override)`. -7. Apply existing enrichment, after the override merge (LCORE-3370): - `enrich_azure_entra_id_inference`, `enrich_byok_rag`, `enrich_solr`, - `enrich_vector_store`. -8. `dedupe_providers_vector_io` again. -9. Return the final dict. +7. `dedupe_providers_vector_io` again for good measure. +8. Return the final dict. **`_load_library_client` fork point** (in `src/client/ogx.py`). The check is "is there a synthesis input?", which spans the root `inference.providers` diff --git a/docs/devel_doc/container_orchestration.md b/docs/devel_doc/container_orchestration.md index 9d695cc01..30f2a09c6 100644 --- a/docs/devel_doc/container_orchestration.md +++ b/docs/devel_doc/container_orchestration.md @@ -298,6 +298,9 @@ The Makefile passes these environment variables to the OGX container: #### OKP/Solr RAG Configuration +> [!WARNING] +> **Deprecated:** The Solr vector provider used by OKP RAG is deprecated. + **For OKP (Offline Knowledge Portal) RAG:** - `RH_SERVER_OKP`: OKP server URL (e.g., `http://localhost:8081`) - `SOLR_URL`: Solr server URL diff --git a/docs/devel_doc/openapi.json b/docs/devel_doc/openapi.json index 8ea645ece..8777a6a53 100644 --- a/docs/devel_doc/openapi.json +++ b/docs/devel_doc/openapi.json @@ -16286,7 +16286,7 @@ "additionalProperties": false, "type": "object", "title": "OkpConfiguration", - "description": "OKP (Offline Knowledge Portal) provider configuration.\n\nControls provider-specific behaviour for the OKP vector store.\nOnly relevant when ``\"okp\"`` is listed in ``rag.retrieval.inline.sources``\nor ``rag.retrieval.tool.sources``." + "description": "OKP (Offline Knowledge Portal) provider configuration.\n\nDeprecated: The Solr vector provider used by OKP RAG is deprecated.\n\nControls provider-specific behaviour for the OKP vector store.\nOnly relevant when ``\"okp\"`` is listed in ``rag.retrieval.inline.sources``\nor ``rag.retrieval.tool.sources``." }, "OpenAIResponseAnnotationCitation": { "properties": { @@ -19680,7 +19680,7 @@ "okp": { "$ref": "#/components/schemas/OkpConfiguration", "title": "OKP configuration", - "description": "OKP provider settings. Only used when 'okp' is listed in retrieval.inline.sources or retrieval.tool.sources." + "description": "Deprecated: The Solr vector provider used by OKP RAG is deprecated. OKP provider settings. Only used when 'okp' is listed in retrieval.inline.sources or retrieval.tool.sources." }, "retrieval": { "$ref": "#/components/schemas/RetrievalConfiguration", @@ -22073,7 +22073,7 @@ "additionalProperties": false, "type": "object", "title": "SolrVectorSearchRequest", - "description": "LCORE Solr inline RAG options for vector_io.query (mode and provider filters).\n\nAttributes:\n mode: Solr vector_io search mode. When omitted, the configured OKP default is used.\n filters: Solr provider filter payload passed through as params['solr'].\n\nLegacy clients may send a plain JSON object with filter keys only;\nthat object is accepted as filters with mode unset (server default applies)." + "description": "LCORE Solr inline RAG options for vector_io.query (mode and provider filters).\n\nDeprecated: The Solr vector provider used by OKP RAG is deprecated.\n\nAttributes:\n mode: Solr vector_io search mode. When omitted, the configured OKP default is used.\n filters: Solr provider filter payload passed through as params['solr'].\n\nLegacy clients may send a plain JSON object with filter keys only;\nthat object is accepted as filters with mode unset (server default applies)." }, "SplunkConfiguration": { "properties": { diff --git a/docs/migrations/v0.7.0.md b/docs/migrations/v0.7.0.md index 382ecaf4d..a6fdceeba 100644 --- a/docs/migrations/v0.7.0.md +++ b/docs/migrations/v0.7.0.md @@ -1,5 +1,8 @@ # Migrating to v0.7.0 +> [!WARNING] +> **Deprecated:** The Solr vector provider used by OKP RAG is deprecated. + ## Table of Contents * [RAG Configuration](#rag-configuration) @@ -27,6 +30,8 @@ API response fields were renamed without backward-compatible aliases. ```yaml llama_stack: url: http://localhost:8321 + config: + profile: default ``` **After (v0.7.0 — preferred):** @@ -34,6 +39,8 @@ llama_stack: ```yaml ogx: url: http://localhost:8321 + config: + profile: default ``` **Still accepted in v0.7.0 (deprecated):** @@ -41,6 +48,8 @@ ogx: ```yaml llama_stack: url: http://localhost:8321 + config: + profile: default ``` A startup warning is logged when the deprecated `llama_stack` key is used. diff --git a/docs/user_doc/conversation_compaction.md b/docs/user_doc/conversation_compaction.md index e7f5221fa..f29887a60 100644 --- a/docs/user_doc/conversation_compaction.md +++ b/docs/user_doc/conversation_compaction.md @@ -136,10 +136,6 @@ The `buffer_turns` setting specifies a target number of recent turns to preserve When compaction occurs, a marker message (prefixed with `[lightspeed:compaction-summary]`) is appended to the conversation in OGX. This marker serves as a fallback for reconstructing the compacted state in cache-less deployments or after cache eviction. -The marker is internal bookkeeping, not something the user said. It stays in storage, but `GET /v1/conversations/{conversation_id}` leaves it out of the returned chat history, so clients see only the turns the user and the assistant exchanged. `GET /v2/conversations/{conversation_id}` reads the conversation cache, which never holds markers. - -The prefix is reserved. A user message that begins with `[lightspeed:compaction-summary]` cannot be told apart from a marker, so `GET /v1/conversations/{conversation_id}` leaves it out as well, whether or not compaction is enabled. - ### Per-conversation locking Compaction acquires a per-conversation lock to prevent concurrent requests on the same conversation from racing during summarization. If multiple requests arrive simultaneously, they are serialized. The lock is automatically released after processing. @@ -158,7 +154,7 @@ When compaction is disabled (the default), requests that cause the conversation **Does compaction lose information?** -Compaction summarizes older turns, so fine-grained details from early in the conversation may be condensed. The full original conversation history remains stored in OGX and is retrievable via the conversations API; the summaries themselves are not part of that history and are not returned. The LLM simply receives a summary instead of the full transcript for inference. +Compaction summarizes older turns, so fine-grained details from early in the conversation may be condensed. The full original conversation history remains stored in OGX and is retrievable via the conversations API. The LLM simply receives a summary instead of the full transcript for inference. **Does compaction use extra tokens?** diff --git a/docs/user_doc/deployment_guide.md b/docs/user_doc/deployment_guide.md index 6840d33de..e6f3db3c0 100644 --- a/docs/user_doc/deployment_guide.md +++ b/docs/user_doc/deployment_guide.md @@ -182,46 +182,6 @@ synthesizer evolves. -### High-level inference providers - -LLM providers can be declared directly in `lightspeed-stack.yaml`, in the -top-level `inference.providers` section, without writing any OGX -configuration: - -```yaml -ogx: - use_as_library_client: true - config: - baseline: byo-llm -inference: - default_provider: openai - default_model: gpt-4o-mini - providers: - - type: openai - api_key_env: OPENAI_API_KEY - allowed_models: - - gpt-4o-mini -``` - -`api_key_env` names the environment variable that holds the key. The -synthesized configuration contains only a `${env.OPENAI_API_KEY}` reference; -the value is never written to disk. A complete file is in -[examples/lightspeed-stack-unified-byo-llm.yaml](../../examples/lightspeed-stack-unified-byo-llm.yaml). - -`ogx.config.baseline` selects the starting point of the synthesis: - -| Baseline | Meaning | -|---|---| -| `byo-llm` | The built-in baseline without any LLM provider. Declare yours under `inference.providers`. Recommended. | -| `default` | The built-in baseline including a built-in OpenAI provider that is enabled when `OPENAI_API_KEY` is set. | -| `empty` | Starts from an empty configuration. Used by `--migrate-config`. | - -> [!WARNING] -> The built-in OpenAI provider in `baseline: default` is deprecated in -> release 0.7 (a startup warning is logged) and will be removed in release -> 0.8. Set `baseline: byo-llm` and declare your LLM providers under -> `inference.providers`. - ### OGX as a server When this mode is selected, OGX is started as a separate REST API service. All communication with OGX is performed via REST API calls, which means that OGX can run on a separate machine if needed. diff --git a/docs/user_doc/okp_guide.md b/docs/user_doc/okp_guide.md index 0f44294d1..f2736751f 100644 --- a/docs/user_doc/okp_guide.md +++ b/docs/user_doc/okp_guide.md @@ -3,6 +3,9 @@ This document explains how to deploy the Offline Knowledge Portal (OKP) as a RAG source and configure Lightspeed Stack and OGX to use it. You will: +> [!WARNING] +> **Deprecated:** The Solr vector provider used by OKP RAG is deprecated. + * Deploy and verify the OKP Solr service * Configure Lightspeed Stack for OKP (inline or tool RAG) * Install dependencies and launch Lightspeed Stack diff --git a/docs/user_doc/rag_guide.md b/docs/user_doc/rag_guide.md index ee6c43cb5..49a8cb964 100644 --- a/docs/user_doc/rag_guide.md +++ b/docs/user_doc/rag_guide.md @@ -31,6 +31,9 @@ This document explains how to configure and customize your RAG pipeline. You wil Lightspeed Core Stack (LCS) supports two complementary RAG strategies: +> [!WARNING] +> **Deprecated:** The Solr vector provider used by OKP RAG is deprecated. + - **Inline RAG**: context is fetched from BYOK vector stores and/or OKP and injected before the LLM request. No tool calls are required. - **Tool RAG**: the LLM can call the `file_search` tool during generation to retrieve context on demand from BYOK vector stores and/or OKP. @@ -530,4 +533,3 @@ When `vector_store` is configured, `default_provider` overwrites `vector_stores.default_provider_id` and `default_embedding_model` during unified synthesis. Annotation settings are not managed by that enricher — keep them in the OGX baseline/profile or `native_override`. - diff --git a/examples/lightspeed-stack-a2a-state-pg.yaml b/examples/lightspeed-stack-a2a-state-pg.yaml index 337302cad..6a6bfa7f4 100644 --- a/examples/lightspeed-stack-a2a-state-pg.yaml +++ b/examples/lightspeed-stack-a2a-state-pg.yaml @@ -9,9 +9,7 @@ service: access_log: true ogx: use_as_library_client: true - # Unified mode: the run.yaml next to this file is the synthesis profile. - config: - profile: run.yaml + library_client_config_path: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/examples/lightspeed-stack-a2a-state-sqlite.yaml b/examples/lightspeed-stack-a2a-state-sqlite.yaml index 4e6282b68..adff530ba 100644 --- a/examples/lightspeed-stack-a2a-state-sqlite.yaml +++ b/examples/lightspeed-stack-a2a-state-sqlite.yaml @@ -9,9 +9,7 @@ service: access_log: true ogx: use_as_library_client: true - # Unified mode: the run.yaml next to this file is the synthesis profile. - config: - profile: run.yaml + library_client_config_path: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/examples/lightspeed-stack-azure-entraid-lib.yaml b/examples/lightspeed-stack-azure-entraid-lib.yaml index 2091d0da9..7256cb035 100644 --- a/examples/lightspeed-stack-azure-entraid-lib.yaml +++ b/examples/lightspeed-stack-azure-entraid-lib.yaml @@ -12,10 +12,7 @@ ogx: # use_as_library_client: false # Alternative for "as library use" use_as_library_client: true - # Unified mode: azure-run.yaml next to this file is the synthesis profile - # (relative paths resolve against this file's directory). - config: - profile: azure-run.yaml + library_client_config_path: examples/azure-run.yaml # url: http://localhost:8321 # api_key: xyzzy user_data_collection: diff --git a/examples/lightspeed-stack-byok-okp-rag.yaml b/examples/lightspeed-stack-byok-okp-rag.yaml index 156038dba..1d3141f7e 100644 --- a/examples/lightspeed-stack-byok-okp-rag.yaml +++ b/examples/lightspeed-stack-byok-okp-rag.yaml @@ -52,6 +52,7 @@ rag: db_path: /tmp/kb.faiss score_multiplier: 1.2 # Weight for this vector store's results (Inline RAG only) + # DEPRECATED: The Solr vector provider used by OKP RAG is deprecated. # OKP provider settings (only used when 'okp' is listed in retrieval sources) okp: offline: true # true = use parent_id for source URLs, false = use reference_url diff --git a/examples/lightspeed-stack-conversation-cache-pg.yaml b/examples/lightspeed-stack-conversation-cache-pg.yaml index dab1b447e..aecbe728b 100644 --- a/examples/lightspeed-stack-conversation-cache-pg.yaml +++ b/examples/lightspeed-stack-conversation-cache-pg.yaml @@ -8,9 +8,7 @@ service: access_log: true ogx: use_as_library_client: true - # Unified mode: the run.yaml next to this file is the synthesis profile. - config: - profile: run.yaml + library_client_config_path: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/examples/lightspeed-stack-conversation-cache-sqlite.yaml b/examples/lightspeed-stack-conversation-cache-sqlite.yaml index 24fd6b80a..1e766febd 100644 --- a/examples/lightspeed-stack-conversation-cache-sqlite.yaml +++ b/examples/lightspeed-stack-conversation-cache-sqlite.yaml @@ -8,9 +8,7 @@ service: access_log: true ogx: use_as_library_client: true - # Unified mode: the run.yaml next to this file is the synthesis profile. - config: - profile: run.yaml + library_client_config_path: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/examples/lightspeed-stack-shields.yaml b/examples/lightspeed-stack-shields.yaml index 2a9eb4e83..5aa84f0a9 100644 --- a/examples/lightspeed-stack-shields.yaml +++ b/examples/lightspeed-stack-shields.yaml @@ -8,9 +8,7 @@ service: access_log: true ogx: use_as_library_client: true - # Unified mode: the run.yaml next to this file is the synthesis profile. - config: - profile: run.yaml + library_client_config_path: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/examples/lightspeed-stack-skills.yaml b/examples/lightspeed-stack-skills.yaml index af0e10470..f40fd1b2f 100644 --- a/examples/lightspeed-stack-skills.yaml +++ b/examples/lightspeed-stack-skills.yaml @@ -8,9 +8,7 @@ service: access_log: true ogx: use_as_library_client: true - # Unified mode: the run.yaml next to this file is the synthesis profile. - config: - profile: run.yaml + library_client_config_path: run.yaml user_data_collection: feedback_enabled: true feedback_storage: "/tmp/data/feedback" diff --git a/examples/lightspeed-stack-unified-byo-llm.yaml b/examples/lightspeed-stack-unified-byo-llm.yaml deleted file mode 100644 index 0820d1947..000000000 --- a/examples/lightspeed-stack-unified-byo-llm.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Unified single-file configuration: no run.yaml is needed. -# -# Lightspeed Core Stack synthesizes the OGX run configuration at startup from -# this file. `baseline: byo-llm` starts from the built-in baseline without any -# LLM provider ("bring your own LLM"); the providers are declared under the -# top-level `inference.providers` section. -# -# The API key is read from the environment variable named by `api_key_env`; -# its value is never written to disk. -name: Lightspeed Core Service (LCS) -service: - host: localhost - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -ogx: - use_as_library_client: true - config: - baseline: byo-llm -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" -authentication: - module: "noop" -inference: - default_provider: openai - default_model: gpt-4o-mini - providers: - - type: openai - api_key_env: OPENAI_API_KEY - allowed_models: - - gpt-4o-mini diff --git a/src/app/endpoints/conversations_v1.py b/src/app/endpoints/conversations_v1.py index d326ca0c1..001a03e8b 100644 --- a/src/app/endpoints/conversations_v1.py +++ b/src/app/endpoints/conversations_v1.py @@ -46,7 +46,6 @@ from models.database.conversations import ( UserConversation, ) -from utils.conversation_compaction import exclude_marker_items from utils.conversations import ( build_conversation_turns_from_items, get_all_conversation_items, @@ -294,11 +293,9 @@ async def get_conversation_endpoint_handler( # pylint: disable=too-many-locals, conversation_id, ) - # Build conversation turns from items and populate turns metadata. - # Compaction summary markers are stored as user messages, but the - # user never sent them, so they are not part of the history. + # Build conversation turns from items and populate turns metadata chat_history = build_conversation_turns_from_items( - exclude_marker_items(items), db_turns, conversation.created_at + items, db_turns, conversation.created_at ) span.set_attribute("conversations.found", True) diff --git a/src/models/common/query.py b/src/models/common/query.py index 3facfbf9c..fbc297eeb 100644 --- a/src/models/common/query.py +++ b/src/models/common/query.py @@ -137,6 +137,8 @@ def validate_image_attachment(self) -> Self: class SolrVectorSearchRequest(BaseModel): """LCORE Solr inline RAG options for vector_io.query (mode and provider filters). + Deprecated: The Solr vector provider used by OKP RAG is deprecated. + Attributes: mode: Solr vector_io search mode. When omitted, the configured OKP default is used. filters: Solr provider filter payload passed through as params['solr']. diff --git a/src/models/config.py b/src/models/config.py index cedac1ef4..cbd78a148 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -2670,6 +2670,8 @@ def validate_unique_rag_ids(self) -> Self: class OkpConfiguration(ConfigurationBase): """OKP (Offline Knowledge Portal) provider configuration. + Deprecated: The Solr vector provider used by OKP RAG is deprecated. + Controls provider-specific behaviour for the OKP vector store. Only relevant when ``"okp"`` is listed in ``rag.retrieval.inline.sources`` or ``rag.retrieval.tool.sources``. @@ -2730,7 +2732,8 @@ class RagConfiguration(ConfigurationBase): okp: OkpConfiguration = Field( default_factory=OkpConfiguration, title="OKP configuration", - description=f"OKP provider settings. Only used when '{constants.OKP_RAG_ID}' " + description=f"Deprecated: The Solr vector provider used by OKP RAG is deprecated. " + f"OKP provider settings. Only used when '{constants.OKP_RAG_ID}' " "is listed in retrieval.inline.sources or retrieval.tool.sources.", ) diff --git a/tests/e2e/features/conversation-compaction.feature b/tests/e2e/features/conversation-compaction.feature index 0619851ff..fcf48771e 100644 --- a/tests/e2e/features/conversation-compaction.feature +++ b/tests/e2e/features/conversation-compaction.feature @@ -73,7 +73,6 @@ Feature: Conversation compaction | green-harbor | When I use REST API conversation endpoint with conversation_id from above using HTTP GET method Then The status code of the response is 200 - And The conversation history holds 5 turns and no compaction summary marker And The conversation history includes the following user queries | User query | | My OpenShift cluster is named aurora-prod-7 and it runs in the datacenter called north-quarry. Remember both names and reply with OK only. | diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index 663d26bb0..16c3d8e08 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -21,7 +21,6 @@ Feature: Info tests And The body of the response has proper name Lightspeed Core Service (LCS) and version 0.7.0 And The body of the response has ogx version 1.2.5 - Scenario: Check if tools endpoint is working When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 200 diff --git a/tests/e2e/features/steps/conversation_compaction.py b/tests/e2e/features/steps/conversation_compaction.py index 2367fd801..d78ac040b 100644 --- a/tests/e2e/features/steps/conversation_compaction.py +++ b/tests/e2e/features/steps/conversation_compaction.py @@ -19,9 +19,6 @@ from tests.e2e.features.steps.common import get_active_lightspeed_stack_config_basename from tests.e2e.utils.utils import absolute_repo_path, is_prow_environment -COMPACTION_MARKER_PREFIX = "[lightspeed:compaction-summary]" -"""Prefix of the message compaction stores each summary in.""" - def _active_fixture_path(context: Context) -> str: """Resolve the fixture file the last ``The service uses ...`` step applied. @@ -139,35 +136,6 @@ def check_history_includes_user_queries(context: Context) -> None: ) -@then("The conversation history holds {turns:d} turns and no compaction summary marker") -def check_history_holds_no_marker(context: Context, turns: int) -> None: - """Assert the history is the turns that were sent, with no summary marker (LCORE-3909). - - Compaction keeps each summary in the stored conversation as a synthetic - user message that starts with a fixed prefix. The prefix is spelled out - here, not imported, because steps never import from ``src/``. - - The number of turns is checked as well, so the step does not depend on the - prefix alone: a marker is a user message, and every one of them returned - would show up as a turn of its own. - """ - assert context.response is not None, "Request needs to be performed first" - response_json = context.response.json() - assert "chat_history" in response_json, "chat_history not found in response" - chat_history = response_json["chat_history"] - markers = [ - message["content"] - for turn in chat_history - for message in turn.get("messages", []) - if message.get("content", "").startswith(COMPACTION_MARKER_PREFIX) - ] - assert not markers, f"compaction summary markers in the history: {markers!r}" - assert len(chat_history) == turns, ( - f"the history holds {len(chat_history)} turns, expected {turns}: " - f"{[turn.get('messages', []) for turn in chat_history]!r}" - ) - - @then("The streamed response contains a compaction event before the first token") def check_compaction_event_precedes_tokens(context: Context) -> None: """Assert the stream announced compaction before any answer token (R12).""" diff --git a/tests/e2e/features/steps/unified_mode.py b/tests/e2e/features/steps/unified_mode.py index e9541f425..3e110ed2c 100644 --- a/tests/e2e/features/steps/unified_mode.py +++ b/tests/e2e/features/steps/unified_mode.py @@ -1,6 +1,6 @@ """Step definitions for the unified-mode e2e features (LCORE-2343). -Only the startup-log evidence step lives here. Everything else the four +Only the startup-log evidence step lives here. Everything else the five ``unified-mode-*.feature`` files need — applying a configuration, restarting containers, hitting ``readiness`` and ``query`` — resolves through the generic steps, and the configuration validation, migration and synthesis assertions diff --git a/tests/integration/endpoints/test_compaction_conversations_api.py b/tests/integration/endpoints/test_compaction_conversations_api.py deleted file mode 100644 index 5de78795f..000000000 --- a/tests/integration/endpoints/test_compaction_conversations_api.py +++ /dev/null @@ -1,316 +0,0 @@ -"""Integration tests for compaction markers and the conversations API (LCORE-3909). - -Compaction stores each summary in the conversation as a synthetic user message -(a *marker*). The marker has to stay in storage, because it is the fallback -source of truth when no conversation cache is configured, but it is not -something the user said, so the conversations API must not return it. -""" - -# pylint: disable=too-many-arguments -# pylint: disable=too-many-positional-arguments - -from collections.abc import Generator -from datetime import UTC, datetime - -import pytest -from fastapi import Request -from ogx_api.openai_responses import OpenAIResponseMessage -from pytest_mock import AsyncMockType, MockerFixture -from sqlalchemy.orm import Session - -from app.endpoints.conversations_v1 import ( - get_conversation_endpoint_handler as get_conversation_v1, -) -from app.endpoints.conversations_v2 import ( - get_conversation_endpoint_handler as get_conversation_v2, -) -from app.endpoints.query import query_endpoint_handler -from authentication.interface import AuthTuple -from cache.sqlite_cache import SQLiteCache -from configuration import AppConfig -from models.api.requests import QueryRequest -from models.api.responses.successful import ConversationResponse -from models.compaction import ConversationSummary -from models.config import SQLiteDatabaseConfiguration -from models.database.conversations import UserTurn -from tests.integration.conftest import InMemoryConversationStore -from tests.integration.endpoints._compaction_helpers import ( - CONV_ID_LLAMA, - DEFAULT_MODEL_RESPONSE, - DEFAULT_SUMMARY_TEXT, - EXISTING_CONV_ID, - TEST_MODEL, - assert_marker_count, - create_existing_conversation, - enable_compaction, - marker, - msg, -) -from utils.conversation_compaction import MARKER_SENTINEL - -NEW_QUERY = "What else can you help with?" - - -@pytest.fixture(name="conversation_cache") -def conversation_cache_fixture( - test_config: AppConfig, - mocker: MockerFixture, -) -> Generator[SQLiteCache, None, None]: - """Configure an in-memory SQLite conversation cache. - - The v2 conversations endpoints read this cache, and compaction keeps its - summaries in it when one is configured. - """ - test_config.conversation_cache_configuration.type = "sqlite" - cache = SQLiteCache(SQLiteDatabaseConfiguration(db_path=":memory:")) - cache.connect() - cache.initialize_cache() - mocker.patch.object( - type(test_config), - "conversation_cache", - new_callable=mocker.PropertyMock, - return_value=cache, - ) - yield cache - - -def _returned_messages(response: ConversationResponse) -> list[tuple[str, str]]: - """Flatten a conversation response into ``(type, content)`` pairs.""" - return [ - (message.type, message.content) - for turn in response.chat_history - for message in turn.messages - ] - - -def _add_turn_metadata( - db_session: Session, turn_number: int, provider: str, model: str -) -> None: - """Store the metadata row the service keeps for one completed turn.""" - now = datetime.now(UTC) - db_session.add( - UserTurn( - conversation_id=EXISTING_CONV_ID, - turn_number=turn_number, - started_at=now, - completed_at=now, - provider=provider, - model=model, - ) - ) - db_session.commit() - - -def _stored_turns() -> list[OpenAIResponseMessage]: - """Two turns long enough to cross the compaction threshold of the tests.""" - return [ - msg("user", "question one " * 20), - msg("assistant", "answer one " * 20), - msg("user", "question two " * 20), - msg("assistant", "answer two " * 20), - ] - - -def _fake_summarization( - mocker: MockerFixture, mock_query_agent: AsyncMockType, covered_items: int -) -> None: - """Replace the summarization LLM call and set the answer of the main turn.""" - mock_query_agent.model.last_output_items = [ - OpenAIResponseMessage(role="assistant", content=DEFAULT_MODEL_RESPONSE) - ] - mocker.patch( - "utils.conversation_compaction.summarize_chunk", - new_callable=mocker.AsyncMock, - return_value=ConversationSummary( - summary_text=DEFAULT_SUMMARY_TEXT, - summarized_through_turn=covered_items, - token_count=6, - created_at="2026-08-10T00:00:00Z", - model_used=TEST_MODEL, - ), - ) - - -class TestConversationsApiCompactionMarkers: - """Compaction markers stay in storage and out of the conversation history.""" - - @pytest.mark.asyncio - async def test_marker_written_by_a_compaction_is_not_returned( - self, - test_config: AppConfig, - mock_ogx_client: AsyncMockType, - mock_query_agent: AsyncMockType, - mock_conversation_store: InMemoryConversationStore, - test_request: Request, - test_auth: AuthTuple, - patch_db_session: Session, - mocker: MockerFixture, - ) -> None: - """A request that compacts, then a read of the same conversation. - - The marker is whatever the running compaction writes, not one built by - the test, so the check holds for the shape the service produces. - - Verifies: - - the compacting request stored a marker - - the conversation read returns the three real turns and no marker - - the marker is still in storage after the read - """ - _ = mock_ogx_client - - enable_compaction(test_config) - user_id, _, _, _ = test_auth - create_existing_conversation(patch_db_session, user_id) - - stored_turns = _stored_turns() - await mock_conversation_store.create( - conversation_id=CONV_ID_LLAMA, items=stored_turns - ) - _fake_summarization(mocker, mock_query_agent, len(stored_turns)) - - await query_endpoint_handler( - request=test_request, - query_request=QueryRequest( - query=NEW_QUERY, conversation_id=EXISTING_CONV_ID - ), - auth=test_auth, - mcp_headers={}, - ) - assert_marker_count(mock_conversation_store, CONV_ID_LLAMA, 1) - - response = await get_conversation_v1( - request=test_request, - conversation_id=EXISTING_CONV_ID, - auth=test_auth, - ) - - assert _returned_messages(response) == [ - ("user", "question one " * 20), - ("assistant", "answer one " * 20), - ("user", "question two " * 20), - ("assistant", "answer two " * 20), - ("user", NEW_QUERY), - ("assistant", DEFAULT_MODEL_RESPONSE), - ] - assert len(response.chat_history) == 3 - assert_marker_count(mock_conversation_store, CONV_ID_LLAMA, 1) - - @pytest.mark.asyncio - async def test_stored_markers_of_both_shapes_are_not_returned( - self, - test_config: AppConfig, - mock_ogx_client: AsyncMockType, - mock_conversation_store: InMemoryConversationStore, - non_admin_test_request: Request, - test_auth: AuthTuple, - patch_db_session: Session, - ) -> None: - """Conversations that already hold markers need no migration. - - The stored conversation has a marker without a covered-item count and - one with it, the shape LCORE-4219 introduces. Both are left out on - read, and every turn keeps its own metadata: a marker counted as a turn - would hand each turn the metadata of its neighbour. - """ - _ = test_config - _ = mock_ogx_client - - user_id, _, _, _ = test_auth - create_existing_conversation(patch_db_session, user_id) - for turn_number in (1, 2, 3): - _add_turn_metadata( - patch_db_session, - turn_number, - provider=f"provider-{turn_number}", - model=f"model-{turn_number}", - ) - - await mock_conversation_store.create( - conversation_id=CONV_ID_LLAMA, - items=[ - msg("user", "question one"), - msg("assistant", "answer one"), - marker("summary of turn one"), - msg("user", "question two"), - msg("assistant", "answer two"), - msg("user", f"{MARKER_SENTINEL} [covers:5] summary of turns one, two"), - msg("user", "question three"), - msg("assistant", "answer three"), - ], - ) - - response = await get_conversation_v1( - request=non_admin_test_request, - conversation_id=EXISTING_CONV_ID, - auth=test_auth, - ) - - assert _returned_messages(response) == [ - ("user", "question one"), - ("assistant", "answer one"), - ("user", "question two"), - ("assistant", "answer two"), - ("user", "question three"), - ("assistant", "answer three"), - ] - assert [(turn.provider, turn.model) for turn in response.chat_history] == [ - ("provider-1", "model-1"), - ("provider-2", "model-2"), - ("provider-3", "model-3"), - ] - assert_marker_count(mock_conversation_store, CONV_ID_LLAMA, 2) - - @pytest.mark.asyncio - async def test_v2_history_holds_no_marker_after_a_compaction( - self, - test_config: AppConfig, - mock_ogx_client: AsyncMockType, - mock_query_agent: AsyncMockType, - mock_conversation_store: InMemoryConversationStore, - conversation_cache: SQLiteCache, - test_request: Request, - test_auth: AuthTuple, - patch_db_session: Session, - mocker: MockerFixture, - ) -> None: - """The v2 endpoint reads the conversation cache, which never holds a marker. - - With a cache configured, a compaction keeps its summary there as well. - The summary goes to the cache's own summary storage and the turn is - cached with the query as the user sent it, so the v2 history holds the - turn and nothing else. - """ - _ = mock_ogx_client - - enable_compaction(test_config) - user_id, _, _, _ = test_auth - create_existing_conversation(patch_db_session, user_id) - - stored_turns = _stored_turns() - await mock_conversation_store.create( - conversation_id=CONV_ID_LLAMA, items=stored_turns - ) - _fake_summarization(mocker, mock_query_agent, len(stored_turns)) - - await query_endpoint_handler( - request=test_request, - query_request=QueryRequest( - query=NEW_QUERY, conversation_id=EXISTING_CONV_ID - ), - auth=test_auth, - mcp_headers={}, - ) - assert_marker_count(mock_conversation_store, CONV_ID_LLAMA, 1) - summaries = conversation_cache.get_summaries(user_id, CONV_ID_LLAMA, False) - assert [summary.summary_text for summary in summaries] == [DEFAULT_SUMMARY_TEXT] - - response = await get_conversation_v2( - request=test_request, - conversation_id=EXISTING_CONV_ID, - auth=test_auth, - ) - - assert _returned_messages(response) == [ - ("user", NEW_QUERY), - ("assistant", DEFAULT_MODEL_RESPONSE), - ] diff --git a/tests/unit/app/endpoints/test_conversations.py b/tests/unit/app/endpoints/test_conversations.py index b8018cadc..c1898b7d3 100644 --- a/tests/unit/app/endpoints/test_conversations.py +++ b/tests/unit/app/endpoints/test_conversations.py @@ -38,7 +38,6 @@ from models.config import Action from models.database.conversations import UserConversation, UserTurn from tests.unit.utils.auth_helpers import mock_authorization_resolvers -from utils.conversation_compaction import MARKER_SENTINEL from utils.conversations import build_conversation_turns_from_items MOCK_AUTH = ("mock_user_id", "mock_username", False, "mock_token") @@ -760,77 +759,6 @@ async def test_successful_conversation_retrieval( ] assert actual_history == expected_chat_history - @pytest.mark.asyncio - async def test_compaction_markers_are_not_returned( - self, - mocker: MockerFixture, - setup_configuration: AppConfig, - expected_chat_history: list[dict[str, Any]], - dummy_request: Request, - mock_conversation: MockType, - ) -> None: - """Test that compaction summary markers are left out of the chat history. - - Compaction stores each summary as a synthetic user message (LCORE-3909). - Both marker shapes are covered: the one recording a covered-item count - (LCORE-4219) and the older one without it. Every turn also has to keep - its own metadata, which a marker counted as a turn would shift. - """ - mock_authorization_resolvers(mocker) - mocker.patch( - "app.endpoints.conversations_v1.configuration", setup_configuration - ) - mocker.patch("app.endpoints.conversations_v1.check_suid", return_value=True) - - mock_db_turns = [ - create_mock_db_turn( - mocker, 1, "2024-01-01T00:01:00Z", "2024-01-01T00:01:05Z" - ), - create_mock_db_turn( - mocker, 2, "2024-01-01T00:02:00Z", "2024-01-01T00:02:03Z" - ), - ] - mock_database_session( - mocker, query_result=[mock_conversation], db_turns=mock_db_turns - ) - - mock_client = mocker.AsyncMock() - mock_items = mocker.Mock() - mock_items.data = [ - mocker.Mock(type="message", role="user", content="Hello"), - mocker.Mock(type="message", role="assistant", content="Hi there!"), - mocker.Mock( - type="message", - role="user", - content=f"{MARKER_SENTINEL} summary of the greeting", - ), - mocker.Mock(type="message", role="user", content="How are you?"), - mocker.Mock( - type="message", role="assistant", content="I'm doing well, thanks!" - ), - mocker.Mock( - type="message", - role="user", - content=f"{MARKER_SENTINEL} [covers:5] summary of both turns", - ), - ] - mock_items.has_more = False - mock_client.items.list = mocker.AsyncMock(return_value=mock_items) - - mock_client_holder = mocker.patch( - "app.endpoints.conversations_v1.AsyncOgxClientHolder" - ) - mock_client_holder.return_value.get_client.return_value = mock_client - - response = await get_conversation_endpoint_handler( - request=dummy_request, conversation_id=VALID_CONVERSATION_ID, auth=MOCK_AUTH - ) - - actual_history = [ - turn.model_dump(exclude_none=True) for turn in response.chat_history - ] - assert actual_history == expected_chat_history - @pytest.mark.asyncio async def test_retrieve_conversation_returns_none( self, diff --git a/tests/unit/observability/formats/test_rlsapi.py b/tests/unit/observability/formats/test_rlsapi.py index 0ef21e358..1163952c1 100644 --- a/tests/unit/observability/formats/test_rlsapi.py +++ b/tests/unit/observability/formats/test_rlsapi.py @@ -65,7 +65,7 @@ def test_builds_event_with_token_counts(mocker: MockerFixture) -> None: org_id="org1", system_id="sys1", request_id="req_1", - cla_version="CLA/1.0", + cla_version=f"CLA/{__version__}", system_os="RHEL", system_version="9.4", system_arch="x86_64", diff --git a/tests/unit/utils/test_conversation_compaction.py b/tests/unit/utils/test_conversation_compaction.py index b5bea40f7..6756e3b76 100644 --- a/tests/unit/utils/test_conversation_compaction.py +++ b/tests/unit/utils/test_conversation_compaction.py @@ -9,10 +9,7 @@ import pytest from fastapi import HTTPException -from ogx_api.openai_responses import ( - OpenAIResponseInputMessageContentText, - OpenAIResponseMessage, -) +from ogx_api.openai_responses import OpenAIResponseMessage from pytest_mock import MockerFixture from models.common.responses.responses_api_params import ResponsesApiParams @@ -810,70 +807,3 @@ async def holder() -> None: except asyncio.CancelledError: pass assert "conv_cancel" not in cc._conversation_locks - - -# --- markers are bookkeeping, not conversation history (LCORE-3909) --- - - -def test_exclude_marker_items_drops_both_marker_shapes() -> None: - """Markers are dropped whether or not they record a covered-item count. - - LCORE-4219 introduces the shape `` [covers:N] ``; the - markers already stored read `` ``. The turns around - them are returned unchanged and in order. - """ - turns = [ - _msg("user", "first question"), - _msg("assistant", "first answer"), - _msg("user", "second question"), - _msg("assistant", "second answer"), - ] - counted_marker = _msg("user", f"{cc.MARKER_SENTINEL} [covers:2] newer summary") - items = [*turns[:2], _marker("older summary"), *turns[2:], counted_marker] - - assert cc.exclude_marker_items(items) == turns - - -def test_exclude_marker_items_drops_marker_with_content_parts() -> None: - """A marker is recognized when its content is a list of parts, not a string.""" - marker = OpenAIResponseMessage( - role="user", - content=[ - OpenAIResponseInputMessageContentText( - text=f"{cc.MARKER_SENTINEL} summary kept as a content part" - ) - ], - ) - question = _msg("user", "a question") - - assert cc.exclude_marker_items([marker, question]) == [question] - - -def test_exclude_marker_items_keeps_everything_else() -> None: - """Tool items and messages that only mention the sentinel are not markers.""" - items = [ - _msg("user", f"what does {cc.MARKER_SENTINEL} mean?"), - SimpleNamespace(type="function_call", name="lookup"), - _msg("assistant", "it is an internal prefix"), - ] - - assert cc.exclude_marker_items(items) == items - - -def test_exclude_marker_items_keeps_other_roles() -> None: - """Markers are written as user messages, so an answer is never one. - - An answer that happens to start with the sentinel is something the - assistant said, and the history has to show it. - """ - items = [ - _msg("user", "which prefix marks a summary?"), - _msg("assistant", f"{cc.MARKER_SENTINEL} is the prefix"), - ] - - assert cc.exclude_marker_items(items) == items - - -def test_exclude_marker_items_empty() -> None: - """An empty conversation stays empty.""" - assert cc.exclude_marker_items([]) == []