diff --git a/.circleci/config.yml b/.circleci/config.yml index b55277ca..27f4d258 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,14 @@ version: 2.1 commands: + check-dependency-versions: + steps: + - run: + name: Validate Dependency Versions + command: | + python -m pip install requests packaging + python scripts/check_dependencies.py --strict + check-if-tests-needed: steps: - run: @@ -155,6 +163,7 @@ jobs: steps: - checkout - check-if-tests-needed + - check-dependency-versions - pip-install-deps - pip-install-tests-deps - run-tests-with-coverage-report @@ -172,6 +181,7 @@ jobs: steps: - checkout - check-if-tests-needed + - check-dependency-versions - pip-install-deps - pip-install-tests-deps: requirements: "tests/requirements-cassandra.txt" @@ -188,6 +198,7 @@ jobs: steps: - checkout - check-if-tests-needed + - check-dependency-versions - pip-install-deps - pip-install-tests-deps: requirements: "tests/requirements-gevent-starlette.txt" @@ -204,6 +215,7 @@ jobs: steps: - checkout - check-if-tests-needed + - check-dependency-versions - pip-install-deps - pip-install-tests-deps: requirements: "tests/requirements-aws.txt" @@ -247,6 +259,7 @@ jobs: steps: - checkout - check-if-tests-needed + - check-dependency-versions - pip-install-deps - pip-install-tests-deps: requirements: "tests/requirements-kafka.txt" @@ -268,6 +281,7 @@ jobs: steps: - checkout - check-if-tests-needed + - check-dependency-versions - pip-install-deps - pip-install-tests-deps: requirements: "tests/requirements-minimal.txt" diff --git a/.dependency-versions.json b/.dependency-versions.json new file mode 100644 index 00000000..1c168771 --- /dev/null +++ b/.dependency-versions.json @@ -0,0 +1,710 @@ +{ + "schema_version": "1.0", + "last_updated": "2026-05-20T13:44:13.490939", + "emergency_overrides": {}, + "dependencies": { + "coverage": { + "current_approved": "7.14.0", + "pending_versions": [ + { + "version": "7.14.0", + "discovered_at": "2026-05-10T00:00:00", + "eligible_at": "2026-05-15T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pytest": { + "current_approved": "9.0.3", + "pending_versions": [ + { + "version": "9.0.3", + "discovered_at": "2026-04-07T00:00:00", + "eligible_at": "2026-04-12T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pytest-mock": { + "current_approved": "3.15.1", + "pending_versions": [ + { + "version": "3.15.1", + "discovered_at": "2025-09-16T00:00:00", + "eligible_at": "2025-09-21T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "couchbase": { + "current_approved": "4.6.1", + "pending_versions": [ + { + "version": "4.6.1", + "discovered_at": "2026-04-29T00:00:00", + "eligible_at": "2026-05-04T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "aiofiles": { + "current_approved": "25.1.0", + "pending_versions": [ + { + "version": "25.1.0", + "discovered_at": "2025-10-09T00:00:00", + "eligible_at": "2025-10-14T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "aiohttp": { + "current_approved": "3.13.5", + "pending_versions": [ + { + "version": "3.13.5", + "discovered_at": "2026-03-31T00:00:00", + "eligible_at": "2026-04-05T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "aio-pika": { + "current_approved": "9.6.2", + "pending_versions": [ + { + "version": "9.6.2", + "discovered_at": "2026-03-22T00:00:00", + "eligible_at": "2026-03-27T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "boto3": { + "current_approved": "1.43.10", + "pending_versions": [ + { + "version": "1.43.10", + "discovered_at": "2026-05-18T00:00:00", + "eligible_at": "2026-05-23T00:00:00", + "status": "superseded", + "superseded_by": "1.43.11", + "override_reason": null + }, + { + "version": "1.43.11", + "discovered_at": "2026-05-19T00:00:00", + "eligible_at": "2026-05-24T00:00:00", + "status": "waiting", + "superseded_by": null, + "override_reason": null + } + ] + }, + "bottle": { + "current_approved": "0.13.4", + "pending_versions": [ + { + "version": "0.13.4", + "discovered_at": "2025-06-15T00:00:00", + "eligible_at": "2025-06-20T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "celery": { + "current_approved": "5.6.3", + "pending_versions": [ + { + "version": "5.6.3", + "discovered_at": "2026-03-26T00:00:00", + "eligible_at": "2026-03-31T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "Django": { + "current_approved": "6.0.5", + "pending_versions": [ + { + "version": "6.0.5", + "discovered_at": "2026-05-05T00:00:00", + "eligible_at": "2026-05-10T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "fastapi": { + "current_approved": "0.136.1", + "pending_versions": [ + { + "version": "0.136.1", + "discovered_at": "2026-04-23T00:00:00", + "eligible_at": "2026-04-28T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "flask": { + "current_approved": "3.1.3", + "pending_versions": [ + { + "version": "3.1.3", + "discovered_at": "2026-02-19T00:00:00", + "eligible_at": "2026-02-24T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "grpcio": { + "current_approved": "1.80.0", + "pending_versions": [ + { + "version": "1.80.0", + "discovered_at": "2026-03-30T00:00:00", + "eligible_at": "2026-04-04T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "google-cloud-pubsub": { + "current_approved": "2.38.0", + "pending_versions": [ + { + "version": "2.38.0", + "discovered_at": "2026-05-07T00:00:00", + "eligible_at": "2026-05-12T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "google-cloud-storage": { + "current_approved": "3.10.1", + "pending_versions": [ + { + "version": "3.10.1", + "discovered_at": "2026-03-23T00:00:00", + "eligible_at": "2026-03-28T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "legacy-cgi": { + "current_approved": "2.6.4", + "pending_versions": [ + { + "version": "2.6.4", + "discovered_at": "2025-10-27T00:00:00", + "eligible_at": "2025-11-01T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "lxml": { + "current_approved": "6.1.1", + "pending_versions": [ + { + "version": "6.1.1", + "discovered_at": "2026-05-18T00:00:00", + "eligible_at": "2026-05-23T00:00:00", + "status": "waiting", + "superseded_by": null, + "override_reason": null + } + ] + }, + "mock": { + "current_approved": "5.2.0", + "pending_versions": [ + { + "version": "5.2.0", + "discovered_at": "2025-03-03T00:00:00", + "eligible_at": "2025-03-08T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "moto": { + "current_approved": "5.2.1", + "pending_versions": [ + { + "version": "5.2.1", + "discovered_at": "2026-05-10T00:00:00", + "eligible_at": "2026-05-15T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "mysqlclient": { + "current_approved": "2.2.8", + "pending_versions": [ + { + "version": "2.2.8", + "discovered_at": "2026-02-10T00:00:00", + "eligible_at": "2026-02-15T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "psycopg2-binary": { + "current_approved": "2.9.12", + "pending_versions": [ + { + "version": "2.9.12", + "discovered_at": "2026-04-21T00:00:00", + "eligible_at": "2026-04-26T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pika": { + "current_approved": "1.4.0", + "pending_versions": [ + { + "version": "1.4.0", + "discovered_at": "2026-05-06T00:00:00", + "eligible_at": "2026-05-11T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "protobuf": { + "current_approved": "7.34.1", + "pending_versions": [ + { + "version": "7.34.1", + "discovered_at": "2026-03-20T00:00:00", + "eligible_at": "2026-03-25T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + }, + { + "version": "7.35.0", + "discovered_at": "2026-05-19T00:00:00", + "eligible_at": "2026-05-24T00:00:00", + "status": "waiting", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pymongo": { + "current_approved": "4.17.0", + "pending_versions": [ + { + "version": "4.17.0", + "discovered_at": "2026-04-20T00:00:00", + "eligible_at": "2026-04-25T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pyramid": { + "current_approved": "2.1", + "pending_versions": [ + { + "version": "2.1", + "discovered_at": "2026-03-11T00:00:00", + "eligible_at": "2026-03-16T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pytz": { + "current_approved": "2026.2", + "pending_versions": [ + { + "version": "2026.2", + "discovered_at": "2026-05-04T00:00:00", + "eligible_at": "2026-05-09T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "redis": { + "current_approved": "7.4.0", + "pending_versions": [ + { + "version": "7.4.0", + "discovered_at": "2026-03-24T00:00:00", + "eligible_at": "2026-03-29T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "responses": { + "current_approved": "0.26.0", + "pending_versions": [ + { + "version": "0.26.0", + "discovered_at": "2026-02-19T00:00:00", + "eligible_at": "2026-02-24T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "sanic": { + "current_approved": "25.12.0", + "pending_versions": [ + { + "version": "25.12.0", + "discovered_at": "2025-12-31T00:00:00", + "eligible_at": "2026-01-05T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "sqlalchemy": { + "current_approved": "2.0.49", + "pending_versions": [ + { + "version": "2.0.49", + "discovered_at": "2026-04-03T00:00:00", + "eligible_at": "2026-04-08T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "starlette": { + "current_approved": "1.0.0", + "pending_versions": [ + { + "version": "1.0.0", + "discovered_at": "2026-03-22T00:00:00", + "eligible_at": "2026-03-27T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "tornado": { + "current_approved": "6.5.5", + "pending_versions": [ + { + "version": "6.5.5", + "discovered_at": "2026-03-10T00:00:00", + "eligible_at": "2026-03-15T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "tracerite": { + "current_approved": "2.3.1", + "pending_versions": [ + { + "version": "2.3.1", + "discovered_at": "2025-12-30T00:00:00", + "eligible_at": "2026-01-04T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "uvicorn": { + "current_approved": "0.47.0", + "pending_versions": [ + { + "version": "0.47.0", + "discovered_at": "2026-05-14T00:00:00", + "eligible_at": "2026-05-19T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "urllib3": { + "current_approved": "2.7.0", + "pending_versions": [ + { + "version": "2.7.0", + "discovered_at": "2026-05-07T00:00:00", + "eligible_at": "2026-05-12T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "httpx": { + "current_approved": "0.28.1", + "pending_versions": [ + { + "version": "0.28.1", + "discovered_at": "2024-12-06T00:00:00", + "eligible_at": "2024-12-11T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "setuptools": { + "current_approved": "82.0.1", + "pending_versions": [ + { + "version": "82.0.1", + "discovered_at": "2026-03-09T00:00:00", + "eligible_at": "2026-03-14T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "confluent-kafka": { + "current_approved": "2.14.0", + "pending_versions": [ + { + "version": "2.14.0", + "discovered_at": "2026-04-02T00:00:00", + "eligible_at": "2026-04-07T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "kafka-python": { + "current_approved": "2.3.1", + "pending_versions": [ + { + "version": "2.3.1", + "discovered_at": "2026-04-09T00:00:00", + "eligible_at": "2026-04-14T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "kafka-python-ng": { + "current_approved": "2.2.3", + "pending_versions": [ + { + "version": "2.2.3", + "discovered_at": "2024-10-02T00:00:00", + "eligible_at": "2024-10-07T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "cassandra-driver": { + "current_approved": "3.30.0", + "pending_versions": [ + { + "version": "3.30.0", + "discovered_at": "2026-04-16T00:00:00", + "eligible_at": "2026-04-21T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "gevent": { + "current_approved": "26.4.0", + "pending_versions": [ + { + "version": "26.4.0", + "discovered_at": "2026-04-09T00:00:00", + "eligible_at": "2026-04-14T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pytest-timeout": { + "current_approved": "2.4.0", + "pending_versions": [] + }, + "aioamqp": { + "current_approved": "0.15.0", + "pending_versions": [] + }, + "PyMySQL[rsa]": { + "current_approved": "1.0.2", + "pending_versions": [] + }, + "sanic-testing": { + "current_approved": "24.6.0", + "pending_versions": [] + }, + "spyne": { + "current_approved": "2.14.0", + "pending_versions": [] + }, + "autowrapt": { + "current_approved": "1.0", + "pending_versions": [] + }, + "fysom": { + "current_approved": "2.1.6", + "pending_versions": [ + { + "version": "2.1.6", + "discovered_at": "2021-09-10T00:00:00", + "eligible_at": "2021-09-15T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "requests": { + "current_approved": "2.34.2", + "pending_versions": [ + { + "version": "2.34.2", + "discovered_at": "2026-05-14T00:00:00", + "eligible_at": "2026-05-19T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "opentelemetry-api": { + "current_approved": "1.27.0", + "pending_versions": [ + { + "version": "1.42.0", + "discovered_at": "2026-05-19T00:00:00", + "eligible_at": "2026-05-24T00:00:00", + "status": "waiting", + "superseded_by": null, + "override_reason": null + } + ] + }, + "opentelemetry-semantic-conventions": { + "current_approved": "0.48b0", + "pending_versions": [ + { + "version": "0.63b0", + "discovered_at": "2026-05-19T00:00:00", + "eligible_at": "2026-05-24T00:00:00", + "status": "waiting", + "superseded_by": null, + "override_reason": null + } + ] + }, + "typing_extensions": { + "current_approved": "4.15.0", + "pending_versions": [ + { + "version": "4.15.0", + "discovered_at": "2025-08-25T00:00:00", + "eligible_at": "2025-08-30T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pyyaml": { + "current_approved": "6.0.3", + "pending_versions": [ + { + "version": "6.0.3", + "discovered_at": "2025-09-25T00:00:00", + "eligible_at": "2025-09-30T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "psutil": { + "current_approved": "7.2.2", + "pending_versions": [ + { + "version": "7.2.2", + "discovered_at": "2026-01-28T00:00:00", + "eligible_at": "2026-02-02T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + }, + "pre-commit": { + "current_approved": "4.6.0", + "pending_versions": [ + { + "version": "4.6.0", + "discovered_at": "2026-04-21T00:00:00", + "eligible_at": "2026-04-26T00:00:00", + "status": "approved", + "superseded_by": null, + "override_reason": null + } + ] + } + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 02bee134..25ece019 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,8 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ +/lib/ +/lib64/ parts/ sdist/ var/ @@ -101,4 +101,9 @@ ENV/ .vscode # uv (https://docs.astral.sh/uv/) -uv.lock \ No newline at end of file +uv.lock + +# bob +.bob +.agents +skills-lock.json \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82d09d56..b9ff0fc3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,3 +9,12 @@ repos: # Run the formatter. - id: ruff-format types_or: [python, markdown] + +- repo: local + hooks: + - id: check-dependencies + name: Check dependency versions + entry: python scripts/check_dependencies.py --strict + language: system + files: ^(tests/requirements.*\.txt|pyproject\.toml)$ + pass_filenames: false diff --git a/.tekton/.currency/currency-pipeline.yaml b/.tekton/.currency/currency-pipeline.yaml index 0c4ae0f3..6be04c1d 100644 --- a/.tekton/.currency/currency-pipeline.yaml +++ b/.tekton/.currency/currency-pipeline.yaml @@ -20,7 +20,7 @@ spec: workspace: currency-pvc - name: generate-currency-report runAfter: - - clone-repo + - update-version-tracker taskRef: name: generate-currency-report-task workspaces: @@ -34,3 +34,14 @@ spec: workspaces: - name: task-pvc workspace: currency-pvc + - name: update-version-tracker + runAfter: + - clone-repo + params: + - name: revision + value: $(params.revision) + taskRef: + name: update-version-tracker-task + workspaces: + - name: task-pvc + workspace: currency-pvc diff --git a/.tekton/.currency/currency-pipelinerun.yaml b/.tekton/.currency/currency-pipelinerun.yaml index fedc516b..a9cb8e1a 100644 --- a/.tekton/.currency/currency-pipelinerun.yaml +++ b/.tekton/.currency/currency-pipelinerun.yaml @@ -1,14 +1,15 @@ -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: PipelineRun metadata: name: python-currency-pipelinerun spec: + taskRunTemplate: + serviceAccountName: currency-serviceaccount params: - name: revision - value: "main" + value: "cicd/security" pipelineRef: name: python-currency-pipeline - serviceAccountName: currency-serviceaccount workspaces: - name: currency-pvc volumeClaimTemplate: diff --git a/.tekton/.currency/currency-scheduled-eventlistener.yaml b/.tekton/.currency/currency-scheduled-eventlistener.yaml index b410dc94..1b321cb6 100644 --- a/.tekton/.currency/currency-scheduled-eventlistener.yaml +++ b/.tekton/.currency/currency-scheduled-eventlistener.yaml @@ -15,17 +15,18 @@ metadata: name: python-currency-trigger-template spec: resourcetemplates: - - apiVersion: tekton.dev/v1beta1 + - apiVersion: tekton.dev/v1 kind: PipelineRun metadata: generateName: python-currency- spec: + taskRunTemplate: + serviceAccountName: currency-serviceaccount pipelineRef: name: python-currency-pipeline - serviceAccountName: currency-serviceaccount params: - name: revision - value: "main" + value: "cicd/security" workspaces: - name: currency-pvc volumeClaimTemplate: diff --git a/.tekton/.currency/currency-tasks.yaml b/.tekton/.currency/currency-tasks.yaml index 7f5ead15..a7e9f940 100644 --- a/.tekton/.currency/currency-tasks.yaml +++ b/.tekton/.currency/currency-tasks.yaml @@ -94,3 +94,52 @@ spec: git commit -m "chore: Updated Python currency report" git push origin main +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: update-version-tracker-task +spec: + params: + - name: github-token-secret + default: instana-github-api-token + - name: revision + type: string + default: main + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: update-tracker + image: public.ecr.aws/docker/library/python:3.13-trixie + env: + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: $(params.github-token-secret) + key: "GITHUB_TOKEN" + script: | + #!/usr/bin/env bash + set -e + + apt-get update -q && apt-get install -y -q git + + cd /workspace + git clone https://oauth2:${GITHUB_TOKEN}@github.com/instana/python-sensor \ + -b $(params.revision) version-tracker-repo + cd version-tracker-repo + + pip install requests packaging + python scripts/update_version_tracker.py + + git config user.name "Instana CI" + git config user.email "instana-ci@ibm.com" + + if git diff --quiet .dependency-versions.json; then + echo "No version tracker changes to commit" + else + git add .dependency-versions.json + git commit -m "chore: Update dependency version tracker [skip ci]" + git push origin $(params.revision) + echo "Version tracker updated and pushed" + fi diff --git a/.tekton/.currency/resources/requirements.txt b/.tekton/.currency/resources/requirements.txt index e254e8b7..024e28f3 100644 --- a/.tekton/.currency/resources/requirements.txt +++ b/.tekton/.currency/resources/requirements.txt @@ -2,5 +2,5 @@ requests pandas beautifulsoup4 tabulate -kubernetes +kubernetes==29.0.0 packaging diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index 21186298..f350a89d 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -2,6 +2,7 @@ import json import re from datetime import datetime +from pathlib import Path import pandas as pd @@ -11,6 +12,9 @@ from kubernetes import client, config from packaging.version import Version +# Path to the version tracker file (repo root / .dependency-versions.json) +TRACKER_FILE = Path(__file__).resolve().parents[3] / ".dependency-versions.json" + JSON_FILE = "resources/table.json" REPORT_FILE = "docs/report.md" PIP_INDEX_URL = "https://pypi.org/pypi" @@ -26,6 +30,36 @@ def estimate_days_behind(release_date): return (datetime.today().date() - datetime.strptime(release_date, "%Y-%m-%d").date()).days +def load_approved_versions() -> dict: + """Load currently approved versions from the version tracker. + + The version tracker (update_version_tracker.py) enforces a 5-day grace + period: new PyPI versions stay in 'waiting' status until eligible_at + (release_date + 5 days) has passed, then get promoted to 'approved'. + Using current_approved here means the report respects that grace period + automatically. + + Returns: + dict mapping lowercase package name to its currently approved version. + Returns empty dict if the tracker file does not exist yet. + """ + if not TRACKER_FILE.exists(): + print(f"Version tracker not found at {TRACKER_FILE}, falling back to PyPI latest") + return {} + + with open(TRACKER_FILE) as f: + data = json.load(f) + + approved = {} + for pkg_name, pkg_data in data.get("dependencies", {}).items(): + current_approved = pkg_data.get("current_approved") + if current_approved: + approved[pkg_name.lower()] = current_approved + + print(f"Loaded {len(approved)} approved versions from version tracker ({TRACKER_FILE})") + return approved + + def get_upstream_version(dependency, last_supported_version): """Get the latest version available upstream""" last_supported_version_release_date = "Not found" @@ -102,6 +136,10 @@ def get_upstream_version(dependency, last_supported_version): release_time_latest = release_info_latest[-1]["upload_time_iso_8601"] release_date_latest = re.search(r"([\d-]+)T", release_time_latest)[1] + # Handle case where last_supported_version is not found in releases + if last_supported_version not in response_json["releases"]: + return (latest_version, release_date_latest, "Not found") + release_info_last_supported = response_json["releases"][last_supported_version] release_time_last_supported = release_info_last_supported[-1]["upload_time_iso_8601"] release_date_last_supported = re.search(r"([\d-]+)T", release_time_last_supported)[1] @@ -125,14 +163,23 @@ def get_last_supported_version(tekton_ci_output, dependency): pattern, tekton_ci_output, flags=re.I | re.M ) + if last_supported_version is None: + return None + return last_supported_version[1] def is_up_to_date( - last_supported_version, latest_version, last_supported_version_release_date + last_supported_version, benchmark_version, last_supported_version_release_date ): - """Check if the supported package is up-to-date""" - if Version(last_supported_version) >= Version(latest_version): + """Check if the supported package is up-to-date against the benchmark version. + + benchmark_version is either: + - The tracker's current_approved version (grace period already applied by + update_version_tracker.py), or + - The raw PyPI latest version (fallback for untracked packages). + """ + if Version(last_supported_version) >= Version(benchmark_version): up_to_date = "Yes" days_behind = 0 else: @@ -153,15 +200,35 @@ def get_taskruns(namespace, task_name): version = "v1" plural = "taskruns" + configuration = client.Configuration.get_default_copy() + + # Create ApiClient with the configuration + api_client = client.ApiClient(configuration) + # access the custom resource from tekton - tektonV1 = client.CustomObjectsApi() - taskruns = tektonV1.list_namespaced_custom_object( - group, - version, - namespace, - plural, - label_selector=f"{group}/task={task_name}, triggers.tekton.dev/trigger=python-tracer-scheduled-pipeline-triggger", - )["items"] + tektonV1 = client.CustomObjectsApi(api_client) + try: + # Try scheduled pipeline first + taskruns = tektonV1.list_namespaced_custom_object( + group, + version, + namespace, + plural, + label_selector=f"{group}/task={task_name}, triggers.tekton.dev/trigger=python-tracer-scheduled-pipeline-triggger", + )["items"] + + # If no scheduled taskruns found, get all taskruns for this task + if not taskruns: + print(f"No scheduled taskruns found for {task_name}, searching all taskruns...") + taskruns = tektonV1.list_namespaced_custom_object( + group, + version, + namespace, + plural, + label_selector=f"{group}/task={task_name}", + )["items"] + finally: + api_client.close() filtered_taskruns = list(filter(taskrun_filter, taskruns)) filtered_taskruns.sort( @@ -176,11 +243,22 @@ def process_taskrun_logs( ): """Process taskrun logs""" for tr in taskruns: - pod_name = tr["status"]["podName"] + pod_name = tr["status"].get("podName") taskrun_name = tr["metadata"]["name"] - logs = core_v1_client.read_namespaced_pod_log( - pod_name, namespace, container="step-unittest" - ) + + if not pod_name: + print(f"Warning: No podName found for taskrun {taskrun_name}, skipping...") + continue + + print(f"Attempting to read logs from pod: {pod_name}") + + try: + logs = core_v1_client.read_namespaced_pod_log( + pod_name, namespace, container="step-unittest" + ) + except Exception as e: + print(f"Error reading logs from pod {pod_name}: {e}") + continue if "Successfully installed" in logs: print( f"Retrieving container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}.." @@ -207,11 +285,33 @@ def process_taskrun_logs( def get_tekton_ci_output(): """Get the latest successful scheduled tekton pipeline output""" + import os + try: - config.load_incluster_config() - print("Using in-cluster Kubernetes configuration...") - except config.config_exception.ConfigException: - # Fall back to local config if running locally and not inside cluster + token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" + if os.path.exists(token_path): + print(f"ServiceAccount token found at {token_path}") + with open(token_path, 'r') as f: + token = f.read().strip() + print(f"Token length: {len(token)}") + print(f"Token preview: {token[:50]}...") # İlk 50 karakter + + # Manuel olarak configuration oluştur + configuration = client.Configuration() + configuration.host = "https://kubernetes.default.svc" + configuration.verify_ssl = True + configuration.ssl_ca_cert = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + + with open(token_path, 'r') as f: + token = f.read().strip() + configuration.api_key = {"authorization": f"Bearer {token}"} + + client.Configuration.set_default(configuration) + print("Using manually configured in-cluster Kubernetes configuration...") + print(f"API Server: {configuration.host}") + + except Exception as e: + print(f"Error setting up configuration: {e}") config.load_kube_config() print("Using local Kubernetes configuration...") @@ -248,6 +348,13 @@ def main(): items = data["table"] tekton_ci_output = get_tekton_ci_output() + # Load approved versions from version tracker. + # The tracker enforces a 5-day grace period: new PyPI releases stay + # "waiting" for 5 days before being promoted to "approved". By comparing + # the CI's installed version against current_approved (not raw PyPI latest), + # we automatically honour that grace period. + approved_versions = load_approved_versions() + for item in items: package = item["Package name"] @@ -255,27 +362,46 @@ def main(): last_supported_version = get_last_supported_version( tekton_ci_output, package ) - item.update({"Last Supported Version": last_supported_version}) + if last_supported_version is None: + print(f"Warning: Could not find version for {package} in Tekton CI output, using existing value") + last_supported_version = item.get("Last Supported Version", "N/A") + else: + item.update({"Last Supported Version": last_supported_version}) else: last_supported_version = item["Last Supported Version"] - latest_version, release_date, last_supported_version_release_date = ( - get_upstream_version(package, last_supported_version) - ) + # Skip upstream check if version is N/A or invalid, but keep the item in table + if not last_supported_version or last_supported_version == "N/A": + print(f"Warning: No valid version for {package}, keeping existing data") + continue - up_to_date, days_behind = is_up_to_date( - last_supported_version, latest_version, last_supported_version_release_date - ) + try: + latest_version, release_date, last_supported_version_release_date = ( + get_upstream_version(package, last_supported_version) + ) - item.update( - { - "Latest version": latest_version, - "Up-to-date": up_to_date, - "Release date": release_date, - "Latest Version Published At": last_supported_version_release_date, - "Days behind": f"{days_behind} day/s", - } - ) + # Use tracker's approved version as benchmark when available. + # Fall back to raw PyPI latest for packages not yet in the tracker. + approved_version = approved_versions.get(package.lower(), latest_version) + if approved_version != latest_version: + print(f"{package}: using approved version {approved_version} (latest is {latest_version}, still in grace period)") + + up_to_date, days_behind = is_up_to_date( + last_supported_version, approved_version, last_supported_version_release_date + ) + + item.update( + { + "Latest version": latest_version, + "Up-to-date": up_to_date, + "Release date": release_date, + "Latest Version Published At": last_supported_version_release_date, + "Days behind": f"{days_behind} day/s", + } + ) + except Exception as e: + print(f"Warning: Error processing {package}: {e}, keeping existing data") + continue # Create a DataFrame from the list of dictionaries df = pd.DataFrame(items) diff --git a/.tekton/pipeline.yaml b/.tekton/pipeline.yaml index a74ef6be..b6b6e183 100644 --- a/.tekton/pipeline.yaml +++ b/.tekton/pipeline.yaml @@ -28,10 +28,19 @@ spec: workspaces: - name: task-pvc workspace: python-tracer-ci-pipeline-pvc + - name: check-dependencies + displayName: "Validate dependency versions (5-day grace period)" + runAfter: + - clone + taskRef: + name: python-tracer-check-dependencies-task + workspaces: + - name: task-pvc + workspace: python-tracer-ci-pipeline-pvc - name: unittest-default displayName: "Python $(params.imageDigest)" runAfter: - - clone + - check-dependencies matrix: params: - name: imageDigest @@ -45,7 +54,7 @@ spec: workspace: python-tracer-ci-pipeline-pvc - name: unittest-cassandra runAfter: - - clone + - check-dependencies params: - name: imageDigest value: $(params.py-312-imageDigest) @@ -56,7 +65,7 @@ spec: workspace: python-tracer-ci-pipeline-pvc - name: unittest-gevent-starlette runAfter: - - clone + - check-dependencies params: - name: imageDigest value: $(params.py-313-imageDigest) @@ -67,7 +76,7 @@ spec: workspace: python-tracer-ci-pipeline-pvc - name: unittest-kafka runAfter: - - clone + - check-dependencies params: - name: imageDigest value: $(params.py-313-imageDigest) diff --git a/.tekton/pipelinerun.yaml b/.tekton/pipelinerun.yaml index c77b6520..2c28c397 100644 --- a/.tekton/pipelinerun.yaml +++ b/.tekton/pipelinerun.yaml @@ -3,6 +3,8 @@ kind: PipelineRun metadata: name: python-tracer-ci-pipeline-run spec: + taskRunTemplate: + serviceAccountName: default params: - name: revision value: "tekton" diff --git a/.tekton/run_unittests.sh b/.tekton/run_unittests.sh index d4e5103d..b353500f 100755 --- a/.tekton/run_unittests.sh +++ b/.tekton/run_unittests.sh @@ -17,7 +17,7 @@ PYTHON_MINOR_VERSION="$(echo "${PYTHON_VERSION}" | cut -d'.' -f 2)" case "${TEST_CONFIGURATION}" in default) - [ "${PYTHON_MINOR_VERSION}" -eq "14" ] && export REQUIREMENTS='requirements-pre314.txt' || export REQUIREMENTS='requirements.txt' + [ "${PYTHON_MINOR_VERSION}" -eq "15" ] && export REQUIREMENTS='requirements-pre315.txt' || export REQUIREMENTS='requirements.txt' export TESTS=('tests') ;; cassandra) export REQUIREMENTS='requirements-cassandra.txt' diff --git a/.tekton/task.yaml b/.tekton/task.yaml index f6b21a05..f50aa751 100644 --- a/.tekton/task.yaml +++ b/.tekton/task.yaml @@ -282,3 +282,21 @@ spec: workingDir: /workspace/python-sensor/ command: - /workspace/python-sensor/.tekton/run_unittests.sh +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: python-tracer-check-dependencies-task +spec: + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: check-dependencies + image: public.ecr.aws/docker/library/python:3.13-trixie + workingDir: /workspace/python-sensor + script: | + #!/usr/bin/env bash + set -e + pip install requests packaging + python scripts/check_dependencies.py --strict diff --git a/scripts/check_dependencies.py b/scripts/check_dependencies.py new file mode 100755 index 00000000..c5cd3d23 --- /dev/null +++ b/scripts/check_dependencies.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# (c) Copyright IBM Corp. 2026 + +""" +Check and validate dependency versions +""" + +import argparse +import sys +from pathlib import Path + +# Add lib to path +_SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(_SCRIPT_DIR)) + +from lib.version_tracker import VersionTracker # noqa: E402 +from lib.validator import DependencyValidator, ValidationError # noqa: E402 +from lib.requirements_parser import ( # noqa: E402 + parse_requirements_txt, + find_all_requirements_files, + parse_pyproject_toml, +) + + +def main(): + parser = argparse.ArgumentParser(description="Check dependency versions") + parser.add_argument( + "--strict", action="store_true", help="Fail on validation errors" + ) + parser.add_argument( + "--pyproject", + type=Path, + default=Path("pyproject.toml"), + help="Path to pyproject.toml", + ) + parser.add_argument( + "--tracker", + type=Path, + default=Path(".dependency-versions.json"), + help="Path to version tracker file", + ) + parser.add_argument( + "--status", action="store_true", help="Show status of all dependencies" + ) + + args = parser.parse_args() + + # Initialize tracker + tracker = VersionTracker(args.tracker) + + if args.status: + # Show status + print("Dependency Status:") + print("=" * 80) + + for name, dep_data in tracker.data["dependencies"].items(): + print(f"\n{name}:") + print(f" Current approved: {dep_data['current_approved']}") + + if dep_data.get("pending_versions"): + print(" Pending versions:") + for pv in dep_data["pending_versions"]: + print(f" - {pv['version']} ({pv['status']})") + if pv.get("eligible_at"): + print(f" Eligible: {pv['eligible_at']}") + + if tracker.data.get("emergency_overrides"): + print("\n" + "=" * 80) + print("Emergency Overrides:") + for name, override in tracker.data["emergency_overrides"].items(): + print(f"\n{name}:") + print(f" Version: {override['version']}") + print(f" Reason: {override['reason']}") + print(f" Approved by: {override['approved_by']}") + print(f" Expires: {override['expires_at']}") + + return 0 + + # Parse dependencies from requirements files + dependencies = {} + req_files = find_all_requirements_files(Path.cwd()) + for req_file in req_files: + print(f"Parsing {req_file.relative_to(Path.cwd())}...") + req_deps = parse_requirements_txt(req_file) + dependencies.update(req_deps) + + # Parse dependencies from pyproject.toml + if args.pyproject.exists(): + print(f"Parsing {args.pyproject}...") + pyproject_deps = parse_pyproject_toml(args.pyproject) + dependencies.update(pyproject_deps) + print(f" Found {len(pyproject_deps)} dependencies in pyproject.toml") + + if not dependencies: + print("No dependencies found") + return 0 + + # Validate + validator = DependencyValidator(tracker) + + try: + valid = validator.validate_all(dependencies, strict=args.strict) + + if valid: + print(f"✅ All {len(dependencies)} dependencies validated successfully") + return 0 + else: + print("⚠️ Some dependencies have warnings") + return 1 if args.strict else 0 + + except ValidationError as e: + print(f"❌ Validation failed:\n\n{e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) + +# Made with Bob diff --git a/scripts/emergency_override.py b/scripts/emergency_override.py new file mode 100755 index 00000000..1274cde9 --- /dev/null +++ b/scripts/emergency_override.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# (c) Copyright IBM Corp. 2026 + +""" +Emergency override management for critical security updates +""" + +import argparse +import sys +from pathlib import Path + +# Add lib to path +_SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(_SCRIPT_DIR)) + +from lib.version_tracker import VersionTracker # noqa: E402 + + +AUTHORIZED_APPROVERS = ["security-team", "tech-lead", "release-manager"] + + +def add_override(args): + """Add emergency override""" + if args.approved_by not in AUTHORIZED_APPROVERS: + print(f"❌ Error: '{args.approved_by}' is not authorized") + print(f"Authorized approvers: {', '.join(AUTHORIZED_APPROVERS)}") + return 1 + + tracker = VersionTracker(args.tracker) + + tracker.add_emergency_override( + package_name=args.package, + version=args.version, + reason=args.reason, + approved_by=args.approved_by, + ticket_url=args.ticket, + expires_in_days=args.expires_in_days, + ) + + tracker.save() + + print(f"✅ Emergency override created for {args.package} {args.version}") + print(f" Reason: {args.reason}") + print(f" Approved by: {args.approved_by}") + print(f" Expires in: {args.expires_in_days} days") + if args.ticket: + print(f" Ticket: {args.ticket}") + + return 0 + + +def list_overrides(args): + """List active overrides""" + tracker = VersionTracker(args.tracker) + + if not tracker.data.get("emergency_overrides"): + print("No active emergency overrides") + return 0 + + print("Active Emergency Overrides:") + print("=" * 80) + + for name, override_data in tracker.data["emergency_overrides"].items(): + from lib.models import EmergencyOverride + + override = EmergencyOverride.from_dict(override_data) + + status = "EXPIRED" if override.is_expired() else "ACTIVE" + print(f"\n{name} ({status}):") + print(f" Version: {override.version}") + print(f" Reason: {override.reason}") + print(f" Approved by: {override.approved_by}") + print( + f" Approved at: {override.approved_at.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) + print(f" Expires at: {override.expires_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + if override.ticket_url: + print(f" Ticket: {override.ticket_url}") + + return 0 + + +def remove_override(args): + """Remove emergency override""" + tracker = VersionTracker(args.tracker) + + if args.package not in tracker.data.get("emergency_overrides", {}): + print(f"❌ No override found for {args.package}") + return 1 + + tracker.remove_emergency_override(args.package) + tracker.save() + + print(f"✅ Emergency override removed for {args.package}") + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Manage emergency overrides") + parser.add_argument( + "--tracker", + type=Path, + default=Path(".dependency-versions.json"), + help="Path to version tracker file", + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + # Add command + add_parser = subparsers.add_parser("add", help="Add emergency override") + add_parser.add_argument("package", help="Package name") + add_parser.add_argument("version", help="Version to approve") + add_parser.add_argument("--reason", required=True, help="Reason for override") + add_parser.add_argument("--approved-by", required=True, help="Approver name") + add_parser.add_argument("--ticket", help="Ticket URL") + add_parser.add_argument( + "--expires-in-days", type=int, default=7, help="Days until expiry" + ) + + # List command + subparsers.add_parser("list", help="List active overrides") + + # Remove command + remove_parser = subparsers.add_parser("remove", help="Remove override") + remove_parser.add_argument("package", help="Package name") + + args = parser.parse_args() + + if args.command == "add": + return add_override(args) + elif args.command == "list": + return list_overrides(args) + elif args.command == "remove": + return remove_override(args) + + return 1 + + +if __name__ == "__main__": + sys.exit(main()) + +# Made with Bob diff --git a/scripts/lib/__init__.py b/scripts/lib/__init__.py new file mode 100644 index 00000000..bafe6917 --- /dev/null +++ b/scripts/lib/__init__.py @@ -0,0 +1,9 @@ +# (c) Copyright IBM Corp. 2026 + +""" +Dependency security management library +""" + +__version__ = "1.0.0" + +# Made with Bob diff --git a/scripts/lib/models.py b/scripts/lib/models.py new file mode 100644 index 00000000..6d95467a --- /dev/null +++ b/scripts/lib/models.py @@ -0,0 +1,116 @@ +# (c) Copyright IBM Corp. 2026 + +""" +Data models for dependency version tracking +""" + +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Optional + + +class VersionStatus(str, Enum): + """Status of a pending version""" + + WAITING = "waiting" + APPROVED = "approved" + SUPERSEDED = "superseded" + EMERGENCY_OVERRIDE = "emergency_override" + + +@dataclass +class PendingVersion: + """A version waiting for approval""" + + version: str + discovered_at: datetime + eligible_at: datetime + status: VersionStatus = VersionStatus.WAITING + superseded_by: Optional[str] = None + override_reason: Optional[str] = None + + def to_dict(self) -> dict: + return { + "version": self.version, + "discovered_at": self.discovered_at.isoformat(), + "eligible_at": self.eligible_at.isoformat(), + "status": self.status.value, + "superseded_by": self.superseded_by, + "override_reason": self.override_reason, + } + + @classmethod + def from_dict(cls, data: dict) -> "PendingVersion": + return cls( + version=data["version"], + discovered_at=datetime.fromisoformat(data["discovered_at"]), + eligible_at=datetime.fromisoformat(data["eligible_at"]), + status=VersionStatus(data["status"]), + superseded_by=data.get("superseded_by"), + override_reason=data.get("override_reason"), + ) + + +@dataclass +class EmergencyOverride: + """Emergency override for critical security updates""" + + version: str + reason: str + approved_by: str + approved_at: datetime + expires_at: datetime + ticket_url: Optional[str] = None + + def is_expired(self) -> bool: + return datetime.utcnow() >= self.expires_at + + def to_dict(self) -> dict: + return { + "version": self.version, + "reason": self.reason, + "approved_by": self.approved_by, + "approved_at": self.approved_at.isoformat(), + "expires_at": self.expires_at.isoformat(), + "ticket_url": self.ticket_url, + } + + @classmethod + def from_dict(cls, data: dict) -> "EmergencyOverride": + return cls( + version=data["version"], + reason=data["reason"], + approved_by=data["approved_by"], + approved_at=datetime.fromisoformat(data["approved_at"]), + expires_at=datetime.fromisoformat(data["expires_at"]), + ticket_url=data.get("ticket_url"), + ) + + +@dataclass +class DependencyInfo: + """Information about a dependency""" + + name: str + current_approved: str + pending_versions: list[PendingVersion] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "current_approved": self.current_approved, + "pending_versions": [v.to_dict() for v in self.pending_versions], + } + + @classmethod + def from_dict(cls, name: str, data: dict) -> "DependencyInfo": + return cls( + name=name, + current_approved=data["current_approved"], + pending_versions=[ + PendingVersion.from_dict(v) for v in data.get("pending_versions", []) + ], + ) + + +# Made with Bob diff --git a/scripts/lib/pypi_client.py b/scripts/lib/pypi_client.py new file mode 100644 index 00000000..d431a34f --- /dev/null +++ b/scripts/lib/pypi_client.py @@ -0,0 +1,133 @@ +# (c) Copyright IBM Corp. 2026 + +""" +PyPI API client for fetching package information +Uses existing get_upstream_version from generate_report.py +""" + +import re +import sys +from pathlib import Path +from typing import Optional, Tuple +from packaging import version + +# Import from existing currency script +sys.path.insert( + 0, str(Path(__file__).parent.parent.parent / ".tekton" / ".currency" / "scripts") +) +try: + from generate_report import get_upstream_version + + USING_GENERATE_REPORT = True +except ImportError: + USING_GENERATE_REPORT = False + import requests + + def get_upstream_version( + dependency: str, last_supported_version: str + ) -> Tuple[str, str, str]: + """Fallback implementation using PyPI API""" + PIP_INDEX_URL = "https://pypi.org/pypi" + response = requests.get(f"{PIP_INDEX_URL}/{dependency}/json") + response_json = response.json() + + latest_version = response_json["info"]["version"] + release_info_latest = response_json["releases"][latest_version] + release_time_latest = release_info_latest[-1]["upload_time_iso_8601"] + release_date_latest_match = re.search(r"([\d-]+)T", release_time_latest) + release_date_latest = ( + release_date_latest_match[1] if release_date_latest_match else "" + ) + + release_date_last_supported = "" + if last_supported_version in response_json["releases"]: + release_info_last_supported = response_json["releases"][ + last_supported_version + ] + if release_info_last_supported: + release_time_last_supported = release_info_last_supported[-1][ + "upload_time_iso_8601" + ] + release_date_match = re.search( + r"([\d-]+)T", release_time_last_supported + ) + release_date_last_supported = ( + release_date_match[1] if release_date_match else "" + ) + + return ( + latest_version, + release_date_latest, + release_date_last_supported, + ) + + +class PyPIClient: + """Client for interacting with PyPI API - uses existing generate_report functions""" + + def __init__(self, timeout: int = 10): + self.timeout = timeout + + def get_latest_version( + self, package_name: str, current_version: str + ) -> Optional[str]: + """ + Get the latest version of a package from PyPI + + Args: + package_name: Name of the package + current_version: Current version (needed for generate_report compatibility) + + Returns: + Latest version string or None if package not found + """ + try: + # Use existing function with current version + latest_version, _, _ = get_upstream_version(package_name, current_version) + return latest_version + except Exception as e: + print(f"Warning: Failed to fetch {package_name} from PyPI: {e}") + return None + + def get_release_date(self, package_name: str, version_str: str) -> Optional[str]: + """ + Get the release date of a specific version + + Args: + package_name: Name of the package + version_str: Version string + + Returns: + ISO format date string or None if not found + """ + try: + _, _, release_date = get_upstream_version(package_name, version_str) + return release_date if release_date else None + except Exception as e: + print( + f"Warning: Failed to fetch release date for {package_name} {version_str}: {e}" + ) + return None + + def compare_versions(self, version1: str, version2: str) -> int: + """ + Compare two version strings + + Args: + version1: First version + version2: Second version + + Returns: + -1 if version1 < version2, 0 if equal, 1 if version1 > version2 + """ + v1 = version.parse(version1) + v2 = version.parse(version2) + + if v1 < v2: + return -1 + elif v1 > v2: + return 1 + return 0 + + +# Made with Bob diff --git a/scripts/lib/requirements_parser.py b/scripts/lib/requirements_parser.py new file mode 100644 index 00000000..5c3f3d4a --- /dev/null +++ b/scripts/lib/requirements_parser.py @@ -0,0 +1,208 @@ +# (c) Copyright IBM Corp. 2026 + +""" +Parse requirements from various file formats +""" + +from pathlib import Path +from typing import Dict, Optional + + +def parse_requirement_line(line: str) -> Optional[tuple[str, str]]: + """ + Parse a single requirement line + + Returns: + (package_name, version) or None if not parseable + """ + line = line.strip() + + # Skip comments, empty lines, and -r includes + if not line or line.startswith("#") or line.startswith("-r"): + return None + + # Remove inline comments + if "#" in line: + line = line.split("#")[0].strip() + + # Parse different formats + # Format: package>=version + if ">=" in line: + parts = line.split(">=", 1) + name = parts[0].strip() + version = parts[1].split(";")[0].split("[")[0].strip() + return (name, version) + + # Format: package==version + elif "==" in line: + parts = line.split("==", 1) + name = parts[0].strip() + version = parts[1].split(";")[0].split("[")[0].strip() + return (name, version) + + # Format: package<=version + elif "<=" in line: + parts = line.split("<=", 1) + name = parts[0].strip() + version = parts[1].split(";")[0].split("[")[0].strip() + return (name, version) + + return None + + +def parse_requirements_txt( + file_path: Path, base_dir: Optional[Path] = None +) -> Dict[str, str]: + """ + Parse requirements.txt file + + Args: + file_path: Path to requirements file + base_dir: Base directory for resolving -r includes + + Returns: + Dict of package_name -> version + """ + if base_dir is None: + base_dir = file_path.parent + + dependencies = {} + + if not file_path.exists(): + return dependencies + + with open(file_path, "r") as f: + for line in f: + line = line.strip() + + # Handle -r includes + if line.startswith("-r "): + include_file = line[3:].strip() + include_path = base_dir / include_file + included_deps = parse_requirements_txt(include_path, base_dir) + dependencies.update(included_deps) + continue + + result = parse_requirement_line(line) + if result: + name, version = result + dependencies[name] = version + + return dependencies + + +def parse_pyproject_toml(file_path: Path) -> Dict[str, str]: + """ + Parse pyproject.toml dependencies and optional-dependencies + + Uses a three-tier parsing strategy: + 1. tomllib (Python 3.11+ built-in) + 2. toml (third-party library) + 3. Regex fallback (no dependencies) + + Returns: + Dict of package_name -> version + """ + dependencies = {} + if not file_path.exists(): + return dependencies + + data = None + + # Tier 1: Try built-in tomllib (Python 3.11+) + try: + import tomllib + + with open(file_path, "rb") as f: + data = tomllib.load(f) + except ImportError: + # Tier 2: Try third-party toml library + try: + import toml + + with open(file_path, "r") as f: + data = toml.load(f) + except ImportError: + pass + + # If we successfully parsed with tomllib or toml + if data is not None: + project = data.get("project", {}) + + # Core dependencies + deps = project.get("dependencies", []) + for dep in deps: + res = parse_requirement_line(dep) + if res: + dependencies[res[0]] = res[1] + + # Optional dependencies (dev, test, etc.) + optional_deps = project.get("optional-dependencies", {}) + for group, group_deps in optional_deps.items(): + for dep in group_deps: + res = parse_requirement_line(dep) + if res: + dependencies[res[0]] = res[1] + + return dependencies + + # Tier 3: Robust Regex Fallback + import re + + with open(file_path, "r") as f: + content = f.read() + + # Match dependencies = [...] + deps_match = re.search(r"dependencies\s*=\s*\[(.*?)\]", content, re.DOTALL) + if deps_match: + # Extract quoted strings, handling multi-line + items = re.findall(r'"([^"]+)"', deps_match.group(1)) + for item in items: + # Skip comments + if not item.strip().startswith("#"): + res = parse_requirement_line(item) + if res: + dependencies[res[0]] = res[1] + + # Match all optional-dependencies groups + # Pattern: [project.optional-dependencies] + optional_section = re.search( + r"\[project\.optional-dependencies\](.*?)(?=\[|$)", content, re.DOTALL + ) + + if optional_section: + section_content = optional_section.group(1) + # Find all group = [...] patterns + group_matches = re.finditer( + r"(\w+)\s*=\s*\[(.*?)\]", section_content, re.DOTALL + ) + + for group_match in group_matches: + items = re.findall(r'"([^"]+)"', group_match.group(2)) + for item in items: + if not item.strip().startswith("#"): + res = parse_requirement_line(item) + if res: + dependencies[res[0]] = res[1] + + return dependencies + + +def find_all_requirements_files(base_dir: Path) -> list[Path]: + """ + Find all requirements*.txt files in tests directory + + Returns: + List of requirement file paths + """ + requirements_files = [] + + tests_dir = base_dir / "tests" + if tests_dir.exists(): + for file in tests_dir.glob("requirements*.txt"): + requirements_files.append(file) + + return requirements_files + + +# Made with Bob diff --git a/scripts/lib/validator.py b/scripts/lib/validator.py new file mode 100644 index 00000000..fbd07707 --- /dev/null +++ b/scripts/lib/validator.py @@ -0,0 +1,143 @@ +# (c) Copyright IBM Corp. 2026 + +""" +Dependency version validation +""" + +from enum import Enum + +from .version_tracker import VersionTracker +from .pypi_client import PyPIClient + + +class ValidationResult(str, Enum): + """Result of version validation""" + + APPROVED = "approved" + APPROVED_EMERGENCY = "approved_emergency" + WAITING = "waiting" + REJECTED = "rejected" + + +class ValidationError(Exception): + """Raised when validation fails""" + + pass + + +class DependencyValidator: + """Validates dependency versions against tracker""" + + def __init__(self, tracker: VersionTracker): + self.tracker = tracker + self.pypi = PyPIClient() + + def validate_version( + self, package_name: str, requested_version: str, strict: bool = True + ) -> ValidationResult: + """ + Validate a dependency version + + Args: + package_name: Name of the package + requested_version: Requested version + strict: If True, raise exception on failure + + Returns: + ValidationResult + + Raises: + ValidationError: If validation fails and strict=True + """ + # Check emergency override first + override = self.tracker.get_emergency_override(package_name) + if override and not override.is_expired(): + if requested_version == override.version: + return ValidationResult.APPROVED_EMERGENCY + else: + msg = ( + f"{package_name}: Emergency override requires {override.version}\n" + f"Reason: {override.reason}\n" + f"Approved by: {override.approved_by}" + ) + if strict: + raise ValidationError(msg) + print(f"Warning: {msg}") + return ValidationResult.REJECTED + + # Get approved version + approved = self.tracker.get_approved_version(package_name) + if not approved: + # Package not tracked yet, initialize it + self.tracker.add_pending_version(package_name, requested_version) + self.tracker.save() + return ValidationResult.APPROVED + + # Compare versions + cmp = self.pypi.compare_versions(requested_version, approved) + + if cmp <= 0: + # Requested version is same or older than approved + return ValidationResult.APPROVED + + # Requested version is newer than approved + dep = self.tracker.get_dependency(package_name) + if dep: + for pv in dep.pending_versions: + if pv.version == requested_version: + from datetime import datetime + + # Strip timezone info to make both datetime objects naive + eligible_naive = pv.eligible_at.replace(tzinfo=None) + now_naive = datetime.utcnow().replace(tzinfo=None) + days_remaining = (eligible_naive - now_naive).days + msg = ( + f"{package_name}: {requested_version} is in waiting period.\n" + f"{days_remaining} days remaining until {pv.eligible_at.strftime('%Y-%m-%d')}.\n" + f"Use approved version {approved} instead." + ) + if strict: + raise ValidationError(msg) + print(f"Warning: {msg}") + return ValidationResult.WAITING + + # Unknown version + msg = ( + f"{package_name}: {requested_version} is not tracked.\n" + f"Use approved version {approved}." + ) + if strict: + raise ValidationError(msg) + print(f"Warning: {msg}") + return ValidationResult.REJECTED + + def validate_all(self, dependencies: dict[str, str], strict: bool = True) -> bool: + """ + Validate all dependencies + + Args: + dependencies: Dict of package_name -> version + strict: If True, raise exception on first failure + + Returns: + True if all valid, False otherwise + """ + all_valid = True + errors = [] + + for package_name, version in dependencies.items(): + try: + result = self.validate_version(package_name, version, strict=strict) + if result in (ValidationResult.WAITING, ValidationResult.REJECTED): + all_valid = False + except ValidationError as e: + errors.append(str(e)) + all_valid = False + + if errors and strict: + raise ValidationError("\n\n".join(errors)) + + return all_valid + + +# Made with Bob diff --git a/scripts/lib/version_tracker.py b/scripts/lib/version_tracker.py new file mode 100644 index 00000000..1d00b54b --- /dev/null +++ b/scripts/lib/version_tracker.py @@ -0,0 +1,206 @@ +# (c) Copyright IBM Corp. 2026 + +""" +Version tracking and management +""" + +import json +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +from .models import DependencyInfo, EmergencyOverride, PendingVersion, VersionStatus +from .pypi_client import PyPIClient + + +class VersionTracker: + """Manages dependency version tracking""" + + SCHEMA_VERSION = "1.0" + WAITING_DAYS = 5 + OVERRIDE_EXPIRY_DAYS = 7 + + def __init__(self, tracker_file: Path): + self.tracker_file = tracker_file + self.data = self._load() + self.pypi = PyPIClient() + + def _load(self) -> dict: + """Load tracker data from file""" + if not self.tracker_file.exists(): + return self._create_empty_tracker() + + with open(self.tracker_file, "r") as f: + return json.load(f) + + def _create_empty_tracker(self) -> dict: + """Create empty tracker structure""" + return { + "schema_version": self.SCHEMA_VERSION, + "last_updated": datetime.utcnow().isoformat(), + "emergency_overrides": {}, + "dependencies": {}, + } + + def save(self): + """Save tracker data to file""" + self.data["last_updated"] = datetime.utcnow().isoformat() + self.tracker_file.parent.mkdir(parents=True, exist_ok=True) + with open(self.tracker_file, "w") as f: + json.dump(self.data, f, indent=2) + + def get_dependency(self, name: str) -> Optional[DependencyInfo]: + """Get dependency info""" + if name not in self.data["dependencies"]: + return None + return DependencyInfo.from_dict(name, self.data["dependencies"][name]) + + def set_dependency(self, dep: DependencyInfo): + """Set dependency info""" + self.data["dependencies"][dep.name] = dep.to_dict() + + def get_emergency_override(self, name: str) -> Optional[EmergencyOverride]: + """Get emergency override for a package""" + if name not in self.data["emergency_overrides"]: + return None + return EmergencyOverride.from_dict(self.data["emergency_overrides"][name]) + + def add_emergency_override( + self, + package_name: str, + version: str, + reason: str, + approved_by: str, + ticket_url: Optional[str] = None, + expires_in_days: int = OVERRIDE_EXPIRY_DAYS, + ): + """Add emergency override""" + override = EmergencyOverride( + version=version, + reason=reason, + approved_by=approved_by, + approved_at=datetime.utcnow(), + expires_at=datetime.utcnow() + timedelta(days=expires_in_days), + ticket_url=ticket_url, + ) + self.data["emergency_overrides"][package_name] = override.to_dict() + + # Update dependency to approved immediately + dep = self.get_dependency(package_name) + if dep: + dep.current_approved = version + # Mark pending version as emergency override + for pv in dep.pending_versions: + if pv.version == version: + pv.status = VersionStatus.EMERGENCY_OVERRIDE + pv.override_reason = reason + self.set_dependency(dep) + + def remove_emergency_override(self, package_name: str): + """Remove emergency override""" + if package_name in self.data["emergency_overrides"]: + del self.data["emergency_overrides"][package_name] + + def cleanup_expired_overrides(self): + """Remove expired emergency overrides""" + expired = [] + for name, override_data in self.data["emergency_overrides"].items(): + override = EmergencyOverride.from_dict(override_data) + if override.is_expired(): + expired.append(name) + + for name in expired: + self.remove_emergency_override(name) + + def add_pending_version( + self, package_name: str, version: str, release_date: Optional[str] = None + ): + """ + Add a new pending version + + Args: + package_name: Name of the package + version: Version string + release_date: ISO format release date from PyPI (optional) + """ + dep = self.get_dependency(package_name) + if not dep: + # Initialize dependency with this version as approved + dep = DependencyInfo( + name=package_name, current_approved=version, pending_versions=[] + ) + self.set_dependency(dep) + return + + # Check if version already exists + for pv in dep.pending_versions: + if pv.version == version: + return + + # Supersede older pending versions + for pv in dep.pending_versions: + if pv.status == VersionStatus.WAITING: + pv.status = VersionStatus.SUPERSEDED + pv.superseded_by = version + + # Add new pending version + now = datetime.utcnow() + + # Use release_date if provided, otherwise use discovery time + if release_date: + try: + # Parse ISO date string (format: YYYY-MM-DD) + discovered_at = datetime.fromisoformat( + release_date.replace("Z", "+00:00") + ) + except (ValueError, AttributeError): + discovered_at = now + else: + discovered_at = now + + pending = PendingVersion( + version=version, + discovered_at=discovered_at, + eligible_at=discovered_at + timedelta(days=self.WAITING_DAYS), + status=VersionStatus.WAITING, + ) + dep.pending_versions.append(pending) + self.set_dependency(dep) + + def approve_eligible_versions(self): + """Approve versions that have passed waiting period""" + for name, dep_data in self.data["dependencies"].items(): + dep = DependencyInfo.from_dict(name, dep_data) + now = datetime.utcnow() + + for pv in dep.pending_versions: + if pv.status == VersionStatus.WAITING and now >= pv.eligible_at: + pv.status = VersionStatus.APPROVED + dep.current_approved = pv.version + + self.set_dependency(dep) + + def check_for_updates(self, package_name: str, current_version: str): + """Check PyPI for new versions""" + latest = self.pypi.get_latest_version(package_name, current_version) + if not latest: + return + + if self.pypi.compare_versions(latest, current_version) > 0: + # Get release date for the latest version + release_date = self.pypi.get_release_date(package_name, latest) + self.add_pending_version(package_name, latest, release_date) + + def get_approved_version(self, package_name: str) -> Optional[str]: + """Get the currently approved version""" + # Check emergency override first + override = self.get_emergency_override(package_name) + if override and not override.is_expired(): + return override.version + + # Return approved version + dep = self.get_dependency(package_name) + return dep.current_approved if dep else None + + +# Made with Bob diff --git a/scripts/update_version_tracker.py b/scripts/update_version_tracker.py new file mode 100755 index 00000000..48993bcd --- /dev/null +++ b/scripts/update_version_tracker.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# (c) Copyright IBM Corp. 2026 + +""" +Update version tracker with latest versions from PyPI +""" + +import argparse +import sys +from pathlib import Path + +# Add lib to path +_SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(_SCRIPT_DIR)) + +from lib.version_tracker import VersionTracker # noqa: E402 +from lib.requirements_parser import ( # noqa: E402 + parse_requirements_txt, + find_all_requirements_files, + parse_pyproject_toml, +) + + +def main(): + parser = argparse.ArgumentParser(description="Update version tracker") + parser.add_argument( + "--pyproject", + type=Path, + default=Path("pyproject.toml"), + help="Path to pyproject.toml", + ) + parser.add_argument( + "--tracker", + type=Path, + default=Path(".dependency-versions.json"), + help="Path to version tracker file", + ) + + args = parser.parse_args() + + # Parse dependencies from requirements files + dependencies = {} + req_files = find_all_requirements_files(Path.cwd()) + for req_file in req_files: + req_deps = parse_requirements_txt(req_file) + dependencies.update(req_deps) + + # Parse dependencies from pyproject.toml + if args.pyproject.exists(): + print(f"Parsing {args.pyproject}...") + pyproject_deps = parse_pyproject_toml(args.pyproject) + dependencies.update(pyproject_deps) + print(f" Found {len(pyproject_deps)} dependencies in pyproject.toml") + + if not dependencies: + print("No dependencies found") + return 0 + + # Initialize tracker + tracker = VersionTracker(args.tracker) + + # Cleanup expired overrides + tracker.cleanup_expired_overrides() + + # Check for updates + print(f"Checking {len(dependencies)} dependencies for updates...") + for package_name, current_version in dependencies.items(): + print(f" Checking {package_name}...", end=" ") + tracker.check_for_updates(package_name, current_version) + print("✓") + + # Approve eligible versions + tracker.approve_eligible_versions() + + # Save + tracker.save() + + print(f"\n✅ Version tracker updated: {args.tracker}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) + +# Made with Bob