diff --git a/AGENTS.md b/AGENTS.md index 7511a3da2bf..ca906366e52 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,12 +160,15 @@ Many tests use expected outputs saved directly in the source tree: make run-website-devserver ``` - Mock vulnerability records are located in [`go/cmd/website-devserver/testdata/`](go/cmd/website-devserver/testdata/). Add or edit `.json` records and `.meta.yaml` companion files to immediately see changes on page refresh. -- **Python Website with Datastore Emulator (Legacy)**: - Run the legacy Python website server against a local Datastore emulator: +- **Run against Cloud Datastore**: + Run the Go website server against production Datastore: ```bash - make run-website-emulator + make run-website + ``` + Or against staging Datastore: + ```bash + make run-website-staging ``` - - Add custom mock testcases inside [`gcp/website/testdata/osv/`](gcp/website/testdata/osv/). ### Local API Server Development (Go-native) - To run the public OSV API server locally using the native Go implementation alongside the ESPv2 proxy (which transcodes HTTP/JSON REST requests to gRPC): diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db8b6940c81..7fe41a7e204 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -186,16 +186,15 @@ make run-website #### Running a local UI instance -For contributors without access to the GCP project, you can use the website emulator which does -not require Google Cloud project access. This emulator uses a local datastore -and loads data from a local directory. +For contributors without access to the GCP project, you can use the website devserver which does +not require Google Cloud project access or a Datastore emulator. It serves the Go website using a live flat mock dataset with hot reloading. ```shell -make run-website-emulator +make run-website-devserver ``` -You can add testcase records to `gcp/website/testdata/osv/` to test odd cases. -See [gcp/website/testdata/osv/README.md](gcp/website/testdata/osv/README.md) +Mock vulnerability records are located in `go/cmd/website-devserver/testdata/`. +See [go/cmd/website-devserver/testdata/README.md](go/cmd/website-devserver/testdata/README.md) for more information on the format of these records. ### Linting and formatting diff --git a/Makefile b/Makefile index 041a32be520..67ee0db2f4c 100644 --- a/Makefile +++ b/Makefile @@ -94,19 +94,10 @@ build-website-frontend: cd website/frontend3 && pnpm install && pnpm run build cd website/blog && hugo --buildFuture -d ../dist/static/blog -run-website: build-website-frontend ## Run local Python website against prod Datastore - cd gcp/website && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python main.py - -run-website-staging: build-website-frontend - cd gcp/website && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities $(run-cmd) python main.py - -run-website-emulator: build-website-frontend ## Run local Python website against emulator - cd gcp/website && $(install-cmd) && DATASTORE_EMULATOR_PORT=5002 $(run-cmd) python frontend_emulator.py - -run-go-website: build-website-frontend ## Run local Go website against prod Datastore +run-website: build-website-frontend ## Run local Go website against prod Datastore cd go && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities go run ./cmd/website -static-dir ../website/dist -docs-dir ../docs -run-go-website-staging: build-website-frontend +run-website-staging: build-website-frontend cd go && GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities go run ./cmd/website -static-dir ../website/dist -docs-dir ../docs run-website-devserver: build-website-frontend ## Run local Go website development server against local mock dataset @@ -117,7 +108,7 @@ stage-website-assets: build-website-frontend cp -r website/dist/* go/cmd/website/dist/ cp docs/osv_service_v1.swagger.json go/cmd/website/docs/ -run-go-website-prod: stage-website-assets +run-website-prod: stage-website-assets cd go && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities go run -tags embedstatic ./cmd/website diff --git a/README.md b/README.md index 72d532f0799..200dc810cc9 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ consists of: | `gcp/datastore` | The datastore index file (`index.yaml`) | | `gcp/functions` | The Cloud Function for publishing PyPI vulnerabilities (maintained, but not developed) | | `gcp/indexer` | The determine version `indexer` | -| `gcp/website` | The backend of the osv.dev web interface, with the frontend in `frontend3`
Blog posts (in `blog`) | +| `gcp/website` | Frontend assets for the osv.dev website (in `frontend3`) and blog posts (in `blog`) | | `gcp/workers/` | Workers for bisection and impact analysis (`worker`, `importer`, `alias`)
`cron/` jobs for database backups and processing oss-fuzz records | -| `go/` | Go module for shared libraries and commands (`cmd/exporter`, `cmd/recordchecker`) | +| `go/` | Go module for shared libraries and commands (`cmd/importer`, `cmd/worker`, `cmd/exporter`, `cmd/website`, `cmd/api`, etc.) | | `osv/` | The core OSV Python library, used in basically all Python services
OSV ecosystem package versioning helpers in `ecosystems/`
Datastore model definitions in `models.py` | | `tools/` | Misc scripts/tools, mostly intended for development (datastore stuff, linting)
The `indexer-api-caller` for indexer calling | | `vulnfeeds/` | Go module for (mostly) the NVD CVE conversion
The Alpine feed converter (`cmd/alpine`)
The Debian feed converter (`tools/debian`, which is written in Python) | diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index 522a39ac740..5142b37fe12 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -396,6 +396,25 @@ steps: args: ['push', '--all-tags', 'gcr.io/oss-vdb/nvd-cve-osv'] waitFor: ['build-nvd-cve-osv', 'cloud-build-queue'] +# Build website frontend assets +- name: 'node:24.18' + entrypoint: 'bash' + args: + - '-c' + - | + corepack enable pnpm + pnpm install --frozen-lockfile --ignore-scripts + pnpm run build:prod + dir: 'gcp/website/frontend3' + id: 'build-frontend3' + waitFor: ['setup'] + +- name: 'gcr.io/oss-vdb/ci' + args: ['hugo', '--buildFuture', '-d', '../dist/static/blog'] + dir: 'gcp/website/blog' + id: 'build-hugo' + waitFor: ['setup'] + # Build/push Website image - name: 'gcr.io/cloud-builders/docker' entrypoint: 'bash' @@ -403,11 +422,16 @@ steps: id: 'pull-website' waitFor: ['setup'] - name: 'gcr.io/cloud-builders/docker' - args: ['buildx', 'build', '--build-arg', 'BUILDKIT_INLINE_CACHE=1', + args: ['buildx', 'build', '-t', 'gcr.io/oss-vdb/osv-website:latest', '-t', 'gcr.io/oss-vdb/osv-website:$COMMIT_SHA', - '-f', 'gcp/website/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/osv-website:latest', '--pull', '.'] + '--target', 'website', + '--build-context', 'bindings=../bindings', + '--build-context', 'website-dist=../gcp/website/dist', + '--build-context', 'docs=../docs', + '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/osv-website:latest', '--pull', '.'] + dir: 'go' id: 'build-website' - waitFor: ['pull-website', 'build-first-package-finder'] + waitFor: ['pull-website', 'build-frontend3', 'build-hugo', 'build-first-package-finder'] - name: 'gcr.io/cloud-builders/docker' args: ['push', '--all-tags', 'gcr.io/oss-vdb/osv-website'] waitFor: ['build-website', 'cloud-build-queue'] diff --git a/deployment/clouddeploy/osv-website/run-prod.yaml b/deployment/clouddeploy/osv-website/run-prod.yaml index 7b5177401ba..a3f566bb8a7 100644 --- a/deployment/clouddeploy/osv-website/run-prod.yaml +++ b/deployment/clouddeploy/osv-website/run-prod.yaml @@ -14,6 +14,10 @@ spec: env: - name: OSV_VULNERABILITIES_BUCKET value: 'osv-vulnerabilities' + - name: OSV_LINTER_BUCKET + value: 'osv-public-import-logs' + - name: OSV_API_URL + value: 'api.osv.dev' - name: REDISHOST value: '10.85.52.228' - name: REDISPORT diff --git a/deployment/clouddeploy/osv-website/run-staging.yaml b/deployment/clouddeploy/osv-website/run-staging.yaml index 6139e197b07..b947fe2c04e 100644 --- a/deployment/clouddeploy/osv-website/run-staging.yaml +++ b/deployment/clouddeploy/osv-website/run-staging.yaml @@ -14,6 +14,10 @@ spec: env: - name: OSV_VULNERABILITIES_BUCKET value: 'osv-test-vulnerabilities' + - name: OSV_LINTER_BUCKET + value: 'osv-test-public-import-logs' + - name: OSV_API_URL + value: 'api.test.osv.dev' - name: REDISHOST value: '10.189.34.180' - name: REDISPORT @@ -28,7 +32,7 @@ spec: secretKeyRef: name: google-oauth-client-secret key: 'latest' - - name: FLASK_SECRET_KEY + - name: SESSION_SECRET_KEY valueFrom: secretKeyRef: name: flask-secret-key diff --git a/go/Dockerfile b/go/Dockerfile index 5ffdfd0fe18..c665e3ea58e 100644 --- a/go/Dockerfile +++ b/go/Dockerfile @@ -22,9 +22,10 @@ # # cd go # docker build -t osv/importer --target importer --build-context bindings=../bindings -f Dockerfile . +# docker build -t osv/website --target website --build-context bindings=../bindings --build-context website-dist=../website/dist --build-context docs=../docs -f Dockerfile . # # Select which service to build using the --target flag (e.g. importer, worker, exporter, -# relations, recordchecker, generatesitemap, custommetrics, gitter, first_package_finder, api). +# relations, recordchecker, generatesitemap, custommetrics, gitter, first_package_finder, api, website). # ==================================================================================== # ======================================================== @@ -148,3 +149,15 @@ RUN CGO_ENABLED=0 go build -o /app/api ./cmd/api/ FROM gcr.io/distroless/static-debian12@sha256:a9fcaedd4c9b59e12dd65d954f0b5044f19b0647a8a3712e77205df9e7b102cd AS api COPY --from=api-build /app/api / ENTRYPOINT ["/api"] + +# ======================================================== +# Target: Website +# ======================================================== +FROM builder AS website-build +COPY --from=website-dist . /workspace/go/cmd/website/dist/ +COPY --from=docs osv_service_v1.swagger.json /workspace/go/cmd/website/docs/ +RUN CGO_ENABLED=0 go build -tags embedstatic -o /app/website ./cmd/website/ + +FROM gcr.io/distroless/static-debian12@sha256:a9fcaedd4c9b59e12dd65d954f0b5044f19b0647a8a3712e77205df9e7b102cd AS website +COPY --from=website-build /app/website / +ENTRYPOINT ["/website"]