diff --git a/.github/workflows/build-static.yml b/.github/workflows/build-static.yml index c488a84..340a6cb 100644 --- a/.github/workflows/build-static.yml +++ b/.github/workflows/build-static.yml @@ -9,6 +9,9 @@ name: build-static on: workflow_call: {} +permissions: + contents: read + jobs: build: name: static ${{ matrix.arch }} @@ -22,13 +25,13 @@ jobs: arch: aarch64 runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 # ~200MB static ECL toolchain, rebuilt only when the version or the # build script changes. runner.arch is in the key because runner.os # is "Linux" on both architectures. - name: Cache static ECL toolchain - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: .cache/ecl-static key: ecl-static-26.5.5-${{ runner.arch }}-${{ hashFiles('scripts/build-static.sh') }} @@ -36,11 +39,15 @@ jobs: - name: Build static binary in Alpine run: | mkdir -p .cache/ecl-static + # alpine:3.22 pinned to its manifest-list digest (covers amd64+arm64); + # dependabot cannot see inside run: blocks, so bump it by hand with + # `docker manifest inspect alpine:3.22` (or the registry API) on upgrades. docker run --rm \ -v "$GITHUB_WORKSPACE:/src" \ -v "$GITHUB_WORKSPACE/.cache/ecl-static:/cache" \ -w /src \ - alpine:3.22 sh scripts/build-static.sh + alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce \ + sh scripts/build-static.sh # the container runs as root; later steps and the cache post-step # run as the unprivileged runner user sudo chown -R "$(id -u):$(id -g)" dist bin .cache/ecl-static @@ -53,7 +60,7 @@ jobs: "./dist/clicklisp-linux-${{ matrix.arch }}" version echo '(select (1))' | "./dist/clicklisp-linux-${{ matrix.arch }}" compile - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: clicklisp-linux-${{ matrix.arch }} path: dist/clicklisp-linux-${{ matrix.arch }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b41053..0aa4261 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: # Unit tests + a full binary build with the distro ECL, on both # architectures. @@ -16,7 +19,7 @@ jobs: runner: [ubuntu-24.04, ubuntu-24.04-arm] runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install ECL run: | sudo apt-get update @@ -40,6 +43,7 @@ jobs: runs-on: ${{ matrix.runner }} env: ECL_VERSION: "26.5.5" + ECL_SHA256: "a01a5bcda8c5b73e59dda3494fd13e5fec5db6aa1dad782c3cc3bb57f1633435" # object-size, shift-base and pointer-overflow fire by design in # ECL's core (tagged fixnums, stack-allocated frames) -- ~120 known # reports. Everything else in UBSan plus all of ASan stays on. @@ -54,14 +58,14 @@ jobs: ASAN_OPTIONS: "detect_leaks=0:detect_stack_use_after_return=0:allow_user_segv_handler=1" UBSAN_OPTIONS: "print_stacktrace=1" steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y libgmp-dev - name: Cache sanitized ECL id: cache-ecl - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ~/ecl-asan # bump the -vN suffix when SAN_CFLAGS or configure flags change @@ -70,6 +74,7 @@ jobs: if: steps.cache-ecl.outputs.cache-hit != 'true' run: | curl -fsSL "https://ecl.common-lisp.dev/static/files/release/ecl-${ECL_VERSION}.tgz" -o ecl.tgz + echo "${ECL_SHA256} ecl.tgz" | sha256sum -c - mkdir ecl-src tar xzf ecl.tgz -C ecl-src --strip-components=1 cd ecl-src diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 9f5e1fb..2562b0f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -3,14 +3,9 @@ name: pages on: push: branches: [main] - paths: ["site/**", "logo.png", ".github/workflows/pages.yml"] + paths: ["site/**", "logo.png", "examples/**", "src/**", "clicklisp.asd", "Makefile", "build/build.lisp", "version.sexp", "scripts/gen-site-data.sh", ".github/workflows/pages.yml"] workflow_dispatch: -permissions: - contents: read - pages: write - id-token: write - # One deploy at a time; let an in-flight production deploy finish. concurrency: group: pages @@ -19,12 +14,19 @@ concurrency: jobs: build: runs-on: ubuntu-24.04 + permissions: + contents: read + pages: write # configure-pages (enablement: true) reads/creates the Pages site via the API steps: - - uses: actions/checkout@v7 - - uses: actions/configure-pages@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 with: enablement: true - - uses: actions/setup-node@v7 + - name: Install ECL + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ecl + - name: Build clicklisp and generate demo data + run: make all site-data + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 22 cache: npm @@ -35,16 +37,19 @@ jobs: cp ../logo.png public/logo.png # keep the site logo in sync with the repo's npm ci npm run build - - uses: actions/upload-pages-artifact@v5 + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 with: path: site/dist deploy: needs: build runs-on: ubuntu-24.04 + permissions: + pages: write + id-token: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - id: deployment - uses: actions/deploy-pages@v5 + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 34b6158..87c928c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,14 +4,13 @@ on: push: tags: ["v*"] -permissions: - contents: write - jobs: check-version: runs-on: ubuntu-24.04 + permissions: + contents: read steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Tag must match version.sexp run: | tag="${GITHUB_REF_NAME#v}" @@ -28,9 +27,11 @@ jobs: release: needs: build runs-on: ubuntu-24.04 + permissions: + contents: write steps: - - uses: actions/checkout@v7 - - uses: actions/download-artifact@v8 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: path: dist merge-multiple: true diff --git a/.gitignore b/.gitignore index b3dc2d6..c9e8845 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ dist/ .claude/CLAUDE.md site/node_modules/ site/dist/ +site/src/generated/ *.tsbuildinfo diff --git a/Makefile b/Makefile index 3d52d9b..ba79255 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ECL ?= ecl BIN := bin/clicklisp SOURCES := clicklisp.asd version.sexp $(wildcard src/*.lisp) -.PHONY: all test smoke clean +.PHONY: all test smoke site-data clean all: $(BIN) @@ -23,7 +23,13 @@ smoke: $(BIN) $(BIN) rules sql --all --load examples/analytics/uk-price-paid.lisp $(BIN) rules sql --all --load examples/analytics/repo-health.lisp $(BIN) rules sql --all --load examples/analytics/hackernews.lisp + $(BIN) rules sql --all --load examples/analytics/github-events.lisp + $(BIN) rules sql --all --load examples/analytics/repo-health.lisp --load examples/analytics/repo-health-playground.lisp + $(BIN) rules json --all --load examples/rules.lisp | python3 -m json.tool > /dev/null @echo smoke OK +site-data: $(BIN) + scripts/gen-site-data.sh + clean: rm -rf bin diff --git a/README.md b/README.md index cd87768..490eadd 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,12 @@ parameters; `CLICKLISP_FORMAT` and `CLICKLISP_PLAY` override the output format and the endpoint. One statement per invocation (the ClickHouse HTTP interface is single-statement). +The same queries run as [live demos](https://cl.clickhouse.com/#demos) +in the browser: the site's deploy workflow builds the binary, compiles +the example libraries with `rules json`, and each demo page sends the +compiled SQL to the playground with typed `param_` bindings — no server +of ours in the path. + ## The query language Operators and clause names are matched by symbol *name*, so forms can be @@ -168,8 +174,15 @@ Rules are validated (compiled) at load time and rendered on demand: clicklisp rules --load rules.lisp # list clicklisp rules sql --load rules.lisp --all # emit SQL for all clicklisp rules sql --load rules.lisp ssh-bruteforce +clicklisp rules json --load rules.lisp --all # machine-readable JSON ``` +`rules json` dumps each rule as JSON — `name`, `description`, `severity`, +`tags`, the extracted `{name:Type}` `params`, compact `sql`, multiline +`sql_pretty`, and the reconstructed `form` — for anything downstream that +wants rules as data; it is what feeds the +[live demos](https://cl.clickhouse.com/#demos) at build time. + See [examples/rules.lisp](examples/rules.lisp). Exit codes: `0` success, `1` bad input (a query that does not compile), @@ -179,7 +192,7 @@ Exit codes: `0` success, `1` bad input (a query that does not compile), `defquery` is `defrule` minus the detection metadata: named queries for any domain, served by the same `rules` commands. -[examples/analytics/](examples/analytics/) has three libraries built on +[examples/analytics/](examples/analytics/) has four libraries built on the playground's example datasets: - [uk-price-paid.lisp](examples/analytics/uk-price-paid.lisp) — UK @@ -189,11 +202,21 @@ the playground's example datasets: - [repo-health.lisp](examples/analytics/repo-health.lisp) — engineering analytics (code churn, bus factor, comment-to-code ratio) over `clickhouse git-import` tables; the whole library is one macro - parameterized by table names, so it runs against *your* repo locally - or the playground's pre-imported ClickHouse and Grafana mirrors. + parameterized by table names, so it runs against *your* repo locally, + and [repo-health-playground.lisp](examples/analytics/repo-health-playground.lisp) + re-invokes it against the playground's pre-imported ClickHouse mirror + (load both files; the second registration wins). - [hackernews.lisp](examples/analytics/hackernews.lisp) — text search, array lambdas, CTE composition, and a query that calls back into a clicklisp executable UDF. +- [github-events.lisp](examples/analytics/github-events.lisp) — a + `defrule` threat pack over 11 billion rows of public GitHub events + (`github.events`): branch-reset storms, mass ref deletion, tag + retargeting, star storms, bulk collaborator adds, commit-hour + anomalies, each tagged with MITRE ATT&CK technique IDs. Two rules + default to historical windows because the dataset's loader stopped + populating `push_size` and tag `CreateEvent`s around October 2025 — + see the file header. Rule files are full Common Lisp, and queries are data — so a macro can generate a family of queries with no copy-paste and no templating @@ -365,6 +388,15 @@ compiling Lisp at `CREATE FUNCTION` time. - **release** — pushing a tag `vX.Y.Z` (which must match `version.sexp`) builds both static binaries and publishes them with SHA256SUMS to a GitHub release. +- **pages** — builds the binary with the distro ECL, generates the demo + data (`make site-data`), builds the site, and deploys it to + [cl.clickhouse.com](https://cl.clickhouse.com) — so the demos are + compiled by the same commit they document. + +Supply-chain posture: every workflow action is pinned to a commit SHA +(dependabot keeps the pins fresh), workflows run with least-privilege +`permissions`, the ECL source tarball is SHA256-verified before it is +built, and the Alpine build image is pinned by digest. ## Layout @@ -377,6 +409,8 @@ src/main.lisp CLI entry point build/build.lisp ECL AOT build (c:build-program; works on static ECL) scripts/build-static.sh static musl build, run inside alpine:3.22 scripts/play.sh pipe SQL from stdin to the public ClickHouse playground +scripts/gen-site-data.sh compile the example libraries to JSON for the site demos examples/ detection rules, analytics libraries, UDFs, server config +site/ cl.clickhouse.com — docs plus live demos of the libraries tests/ zero-dependency harness + suite ``` diff --git a/clicklisp.asd b/clicklisp.asd index 1ef83ef..1257c6d 100644 --- a/clicklisp.asd +++ b/clicklisp.asd @@ -17,6 +17,7 @@ :serial t :components ((:file "tests/harness") (:file "tests/compiler-tests") + (:file "tests/rules-json-tests") (:file "tests/udf-tests")) :perform (test-op (o c) (unless (symbol-call '#:clicklisp/test '#:run-tests) diff --git a/examples/analytics/github-events.lisp b/examples/analytics/github-events.lisp new file mode 100644 index 0000000..66968da --- /dev/null +++ b/examples/analytics/github-events.lisp @@ -0,0 +1,139 @@ +;;;; GitHub events threat pack (github.events on the public playground): +;;;; the defrule idea aimed at 11.1 billion rows of public GitHub activity. +;;;; +;;;; Table choice: github.events, whose sort key (event_type, repo_name, +;;;; toDate(created_at)) matches these rules' event_type and repo_name +;;;; predicates, so they prune parts instead of scanning. Avoid +;;;; github.github_events (same data, worse pruning for these shapes) and +;;;; git.github_events (a View). +;;;; +;;;; clicklisp rules sql --load examples/analytics/github-events.lisp branch-reset-storm \ +;;;; | scripts/play.sh since=2025-01-01T00:00:00 until=2025-01-08T00:00:00 +;;;; clicklisp rules sql --load examples/analytics/github-events.lisp mass-ref-deletion \ +;;;; | scripts/play.sh hours=24 +;;;; clicklisp rules sql --load examples/analytics/github-events.lisp star-storm \ +;;;; | scripts/play.sh +;;;; +;;;; play.sh splices parameter values into the URL verbatim, so write +;;;; DateTime values with the ISO T separator, never a space. +;;;; +;;;; Loader caveat: the feed stopped populating push_size / +;;;; push_distinct_size and stopped recording tag CreateEvents around +;;;; October 2025. branch-reset-storm and tag-retarget therefore only +;;;; fire on historical windows; known-good defaults are in their +;;;; comments. + +;; push_distinct_size = 0 with push_size > 0 means no commit in the push +;; was new to the repo: the ref was pointed at already-pushed commits (a +;; reset/rollback, a cross-branch promotion, a mirror sync). The public +;; feed carries no forced flag, and a rebase force-push arrives as +;; all-new SHAs, so true force-push detection is not possible from this +;; dataset -- this flags actors doing bulk ref resets instead, minus the +;; merge-queue-style [bot] accounts that do it legitimately all day. +;; push_size stopped being populated ~Oct 2025 -- query a historical +;; window, e.g. since=2025-01-01T00:00:00 until=2025-01-08T00:00:00. +(defrule branch-reset-storm + (:description "Actors resetting one repo's refs to already-pushed commits 10+ times in a window" + :severity :medium + :tags (:t1565.001 :github :push)) + (select (actor-login repo-name (as (count) resets)) + :from github.events + :where (and (= event-type "PushEvent") + (>= created-at (param since :date-time)) + (< created-at (param until :date-time)) + (> push-size 0) + (= push-distinct-size 0) + (not (like actor-login "%[bot]%"))) + :group-by (actor-login repo-name) + :having (>= resets 10) + :order-by ((resets :desc)) + :limit 50)) + +;; Branch/tag mass deletion: destructive cleanup, or an account wiping +;; the evidence. Window bound server-side, e.g. hours=24. +(defrule mass-ref-deletion + (:description "Actors with 50+ ref deletions in one repo within {hours:UInt32} hours" + :severity :high + :tags (:t1485 :github)) + (select (actor-login repo-name (as (count) deletions) (as (uniq-exact ref) refs)) + :from github.events + :where (and (= event-type "DeleteEvent") + (>= created-at (- (now) (interval (param hours :uint32) :hours)))) + :group-by (actor-login repo-name) + :having (>= deletions 50) + :order-by ((deletions :desc)) + :limit 50)) + +;; A tag deleted and recreated in the same window now points somewhere +;; else -- the classic release-retarget supply-chain move. Tag +;; CreateEvents stopped ~Oct 2025 -- query a historical window, e.g. +;; since=2025-09-01T00:00:00 until=2025-09-08T00:00:00. +(defrule tag-retarget + (:description "Tags deleted and recreated in the same window (release retarget)" + :severity :critical + :tags (:t1195.002 :github :supply-chain)) + (select (repo-name ref + (as (count-if (= event-type "DeleteEvent")) deleted) + (as (count-if (= event-type "CreateEvent")) recreated)) + :from github.events + :where (and (in event-type (list "DeleteEvent" "CreateEvent")) + (= ref-type "tag") + (>= created-at (param since :date-time)) + (< created-at (param until :date-time))) + :group-by (repo-name ref) + :having (and (>= deleted 1) (>= recreated 1)) + :order-by (((+ deleted recreated) :desc)) + :limit 50)) + +;; Repos gaining 50+ stars in a single hour of the last day: launch-day +;; traffic, or a bought star campaign. No parameters -- run it as is. +(defrule star-storm + (:description "Repos gaining 50+ stars in one hour over the last 24 hours" + :severity :medium + :tags (:t1585.001 :github :stars)) + (select (repo-name (as (to-start-of-hour created-at) hour) + (as (count) stars) (as (uniq-exact actor-login) actors)) + :from github.events + :where (and (= event-type "WatchEvent") + (>= created-at (- (now) (interval 24 :hours)))) + :group-by (repo-name hour) + :having (>= stars 50) + :order-by ((stars :desc)) + :limit 50)) + +;; Five or more collaborators added to one repo in a month: onboarding a +;; team, or a compromised owner handing out write access. +(defrule bulk-collaborator-add + (:description "Repos adding 5+ collaborators in the last 30 days" + :severity :medium + :tags (:t1098 :github :access)) + (select (repo-name (as (count) adds) + (as (group-uniq-array member-login) members)) + :from github.events + :where (and (= event-type "MemberEvent") + (= action "added") + (>= created-at (- (now) (interval 30 :days)))) + :group-by (repo-name) + :having (>= adds 5) + :order-by ((adds :desc)) + :limit 50)) + +;; Hour-of-day profile for one repo's pushes: hours busy since {since} +;; (recent >= 10) yet quiet across all prior history (recent > 2x +;; baseline) suggest activity from someone in the wrong timezone. An +;; empty result is the healthy outcome. E.g. repo=ClickHouse/ClickHouse +;; since=2026-07-15T00:00:00. +(defrule commit-hour-anomaly + (:description "Push hours for a repo that are busy recently but rare historically" + :severity :low + :tags (:t1078 :github :anomaly)) + (select ((as (to-hour created-at) hour) + (as (count-if (>= created-at (param since :date-time))) recent) + (as (- (count) recent) baseline) + (as (bar recent 0 100 20) trend)) + :from github.events + :where (and (= event-type "PushEvent") + (= repo-name (param repo :string))) + :group-by (hour) + :having (and (>= recent 10) (> recent (* 2 baseline))) + :order-by (hour))) diff --git a/examples/analytics/repo-health-playground.lisp b/examples/analytics/repo-health-playground.lisp new file mode 100644 index 0000000..e7e918b --- /dev/null +++ b/examples/analytics/repo-health-playground.lisp @@ -0,0 +1,16 @@ +;;;; The repo-health queries aimed at the public playground's pre-imported +;;;; mirror of the ClickHouse repo (git.clickhouse_* tables). The macro and +;;;; the annotated queries live in repo-health.lisp; load this file AFTER +;;;; it -- def-repo-health deliberately re-registers the same four names, +;;;; and the rule registry keeps the last definition (documented overwrite +;;;; semantics), so the playground tables win: +;;;; +;;;; clicklisp rules sql --all --load examples/analytics/repo-health.lisp \ +;;;; --load examples/analytics/repo-health-playground.lisp +;;;; clicklisp rules sql --load examples/analytics/repo-health.lisp \ +;;;; --load examples/analytics/repo-health-playground.lisp comment-ratio \ +;;;; | scripts/play.sh + +(def-repo-health :commits git.clickhouse-commits + :file-changes git.clickhouse-file-changes + :line-changes git.clickhouse-line-changes) diff --git a/examples/analytics/repo-health.lisp b/examples/analytics/repo-health.lisp index 0e73990..d1a0784 100644 --- a/examples/analytics/repo-health.lisp +++ b/examples/analytics/repo-health.lisp @@ -60,9 +60,7 @@ :order-by (dow hour))))) ;; Local git-import tables by default. To aim at the playground's -;; ClickHouse-repo mirror instead, replace the call below with: -;; -;; (def-repo-health :commits git.clickhouse-commits -;; :file-changes git.clickhouse-file-changes -;; :line-changes git.clickhouse-line-changes) +;; ClickHouse-repo mirror instead, load repo-health-playground.lisp after +;; this file -- it re-registers the same four names against the +;; git.clickhouse_* tables. (def-repo-health) diff --git a/scripts/build-static.sh b/scripts/build-static.sh index e8a938b..759d859 100755 --- a/scripts/build-static.sh +++ b/scripts/build-static.sh @@ -11,7 +11,19 @@ # dependencies, ready for ClickHouse's user_scripts_path. set -eu +# NOTE: editing this script changes its hashFiles() cache key in CI, so +# the static-ECL toolchain rebuilds once per arch (~3 minutes) -- that is +# expected; no manual key bump needed. +# +# Fail closed: overriding ECL_VERSION without a matching ECL_SHA256 would +# silently skip verification of an unexpected tarball. +if [ -n "${ECL_VERSION:-}" ] && [ -z "${ECL_SHA256:-}" ]; then + echo "ERROR: ECL_VERSION is overridden but ECL_SHA256 is not." >&2 + echo "Supply both, e.g. ECL_SHA256=\$(sha256sum ecl-\$ECL_VERSION.tgz)" >&2 + exit 2 +fi ECL_VERSION="${ECL_VERSION:-26.5.5}" +ECL_SHA256="${ECL_SHA256:-a01a5bcda8c5b73e59dda3494fd13e5fec5db6aa1dad782c3cc3bb57f1633435}" ECL_TARBALL="https://ecl.common-lisp.dev/static/files/release/ecl-${ECL_VERSION}.tgz" ECL_PREFIX="/cache/ecl-${ECL_VERSION}-static" ARCH="$(uname -m)" @@ -20,9 +32,12 @@ apk add --no-cache build-base gmp-dev gmp-static curl file if [ ! -x "$ECL_PREFIX/bin/ecl" ]; then echo "=== building static ECL $ECL_VERSION for musl/$ARCH" - rm -rf /tmp/ecl-src + rm -rf /tmp/ecl-src /tmp/ecl.tgz mkdir -p /tmp/ecl-src - curl -fsSL "$ECL_TARBALL" | tar xz -C /tmp/ecl-src --strip-components=1 + curl -fsSL "$ECL_TARBALL" -o /tmp/ecl.tgz + echo "$ECL_SHA256 /tmp/ecl.tgz" | sha256sum -c - + tar xzf /tmp/ecl.tgz -C /tmp/ecl-src --strip-components=1 + rm /tmp/ecl.tgz cd /tmp/ecl-src # In-tree configure (ECL's wrapper does not support external build dirs). # The bundled GMP is ancient (4.2.1); Alpine's gmp-static provides a diff --git a/scripts/gen-site-data.sh b/scripts/gen-site-data.sh new file mode 100755 index 0000000..0e262ba --- /dev/null +++ b/scripts/gen-site-data.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Compile the example analytics rule packs to JSON for the site's demo +# pages. Consumed by `make site-data`; the output directory is +# gitignored, and the pages workflow regenerates it on every deploy so +# the site cannot drift from the compiler. +# +# Environment: +# BIN path to the clicklisp binary (default: bin/clicklisp) +set -eu + +cd "$(dirname "$0")/.." + +BIN="${BIN:-bin/clicklisp}" +if [ ! -x "$BIN" ]; then + echo "ERROR: $BIN not found or not executable; build it first (make all)" >&2 + exit 1 +fi + +mkdir -p site/src/generated + +"$BIN" rules json --all --load examples/analytics/github-events.lisp > site/src/generated/github-threats.json +"$BIN" rules json --all --load examples/analytics/uk-price-paid.lisp > site/src/generated/uk-price-paid.json +"$BIN" rules json --all --load examples/analytics/hackernews.lisp > site/src/generated/hackernews.json +"$BIN" rules json --all --load examples/analytics/repo-health.lisp --load examples/analytics/repo-health-playground.lisp > site/src/generated/repo-health.json diff --git a/site/github-threats/index.html b/site/github-threats/index.html new file mode 100644 index 0000000..176b98b --- /dev/null +++ b/site/github-threats/index.html @@ -0,0 +1,24 @@ + + + + + + + + + clicklisp demos — GitHub events threat pack + + + + + +
+ + + diff --git a/site/hackernews/index.html b/site/hackernews/index.html new file mode 100644 index 0000000..0e0b850 --- /dev/null +++ b/site/hackernews/index.html @@ -0,0 +1,24 @@ + + + + + + + + + clicklisp demos — Hacker News analytics + + + + + +
+ + + diff --git a/site/index.html b/site/index.html index 1874342..44572cb 100644 --- a/site/index.html +++ b/site/index.html @@ -4,6 +4,7 @@ + + + + + + + + + clicklisp demos — Repo health + + + + + +
+ + + diff --git a/site/scripts/check-generated.mjs b/site/scripts/check-generated.mjs new file mode 100644 index 0000000..11ae831 --- /dev/null +++ b/site/scripts/check-generated.mjs @@ -0,0 +1,20 @@ +// Guard for npm run dev / npm run build: the demo pages import build-time +// JSON emitted by the clicklisp binary (gitignored), so a fresh checkout +// fails with a cryptic Rollup "failed to resolve import" unless we say +// what's actually missing. +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SLUGS = ["github-threats", "uk-price-paid", "hackernews", "repo-health"]; + +const siteRoot = dirname(dirname(fileURLToPath(import.meta.url))); +const missing = SLUGS.map((slug) => join(siteRoot, "src", "generated", `${slug}.json`)).filter( + (path) => !existsSync(path) +); + +if (missing.length > 0) { + console.error("demo data missing — run 'make site-data' at the repo root (needs ECL)"); + for (const path of missing) console.error(` missing: ${path}`); + process.exit(1); +} diff --git a/site/src/App.tsx b/site/src/App.tsx index 2fe84cd..d082bf4 100644 --- a/site/src/App.tsx +++ b/site/src/App.tsx @@ -5,6 +5,7 @@ import Language from "./sections/Language"; import Rules from "./sections/Rules"; import Udfs from "./sections/Udfs"; import Playground from "./sections/Playground"; +import Demos from "./sections/Demos"; import GetStarted from "./sections/GetStarted"; import Faq from "./sections/Faq"; import Footer from "./sections/Footer"; @@ -20,6 +21,7 @@ export default function App() { + diff --git a/site/src/demos/DemoLayout.tsx b/site/src/demos/DemoLayout.tsx new file mode 100644 index 0000000..ed0c815 --- /dev/null +++ b/site/src/demos/DemoLayout.tsx @@ -0,0 +1,89 @@ +import { CodeBlock, Icon, Text, Title } from "@clickhouse/click-ui"; +import { CodePane, Eyebrow, REPO_URL, Section, useGitHubStars } from "../components"; +import type { Demo } from "./registry"; +import type { RulesJson } from "./types"; +import RuleCard from "./RuleCard"; + +// Shared chrome + body for the four demo pages. Landing-page anchors must be +// absolute paths ("/#demos") — a bare #hash does not cross pages. +export default function DemoLayout({ demo, data }: Readonly<{ demo: Demo; data: RulesJson }>) { + const stars = useGitHubStars(); + return ( + <> +
+
+
+
+ Live demo + + {demo.title} + +
+ + {demo.blurb} + + + Dataset:{" "} + + {demo.dataset} + + {" · "}rules compiled from{" "} + {demo.exampleFiles.map((href, i) => ( + + {i > 0 && ", "} + + {href.split("/").slice(-1)[0]} + + + ))} + {" "}with clicklisp {data.clicklisp}. Runs in your browser against the read-only public + playground. + +
+
+
Run it yourself from a checkout
+ + + {demo.loadCommand} + + +
+
+ {data.rules.map((rule) => ( + + ))} +
+
+
+ + + ); +} diff --git a/site/src/demos/ResultsTable.tsx b/site/src/demos/ResultsTable.tsx new file mode 100644 index 0000000..296a0bd --- /dev/null +++ b/site/src/demos/ResultsTable.tsx @@ -0,0 +1,78 @@ +import { Badge, Table, Text } from "@clickhouse/click-ui"; +import type { RunResult, RunSummary } from "./playground"; + +const MAX_ROWS = 200; +const MAX_CELL_CHARS = 120; + +function truncate(s: string, max: number): string { + return s.length > max ? `${s.slice(0, max)}…` : s; +} + +function renderCell(value: unknown): string { + if (value === null || value === undefined) return "—"; + if (Array.isArray(value)) { + const joined = value + .map((v) => (v === null || v === undefined ? "—" : typeof v === "object" ? JSON.stringify(v) : String(v))) + .join(", "); + return truncate(joined, MAX_CELL_CHARS); + } + if (typeof value === "object") return JSON.stringify(value); + return String(value); +} + +function humanCount(n: number): string { + if (n >= 1e9) return `${(n / 1e9).toFixed(2)}B`; + if (n >= 1e6) return `${(n / 1e6).toFixed(2)}M`; + if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`; + return String(n); +} + +function humanBytes(n: number): string { + const units = ["B", "KiB", "MiB", "GiB", "TiB"]; + let v = n; + let i = 0; + while (v >= 1024 && i < units.length - 1) { + v /= 1024; + i += 1; + } + return `${i === 0 ? String(v) : v.toFixed(1)} ${units[i]}`; +} + +function StatsRow({ summary }: Readonly<{ summary: RunSummary }>) { + return ( +
+ {summary.readRows !== undefined && ( + + )} + {summary.readBytes !== undefined && ( + + )} + {summary.elapsedNs !== undefined && ( + + )} +
+ ); +} + +export default function ResultsTable({ result }: Readonly<{ result: RunResult }>) { + const shown = result.data.slice(0, MAX_ROWS); + return ( +
+ {result.summary && } + ({ label: m.name }))} + rows={shown.map((row, i) => ({ + id: i, + items: row.map((cell) => ({ label: renderCell(cell) })), + }))} + noDataMessage="No rows — nothing anomalous in this window." + /> + {result.data.length > MAX_ROWS && ( + + showing first {MAX_ROWS} of {result.rows} rows + + )} + + ); +} diff --git a/site/src/demos/RuleCard.tsx b/site/src/demos/RuleCard.tsx new file mode 100644 index 0000000..199c1f2 --- /dev/null +++ b/site/src/demos/RuleCard.tsx @@ -0,0 +1,151 @@ +import { useMemo, useRef, useState } from "react"; +import { Alert, Badge, Button, CodeBlock, Text, TextField, Title } from "@clickhouse/click-ui"; +import { CodePane } from "../components"; +import type { RuleJson } from "./types"; +import type { Demo } from "./registry"; +import { runQuery, type RunResult } from "./playground"; +import ResultsTable from "./ResultsTable"; + +const SEVERITY_STATE = { + critical: "danger", + high: "danger", + medium: "warning", + low: "info", + info: "neutral", +} as const; + +const MAX_PARAM_CHARS = 200; +const MAX_ERROR_CHARS = 2000; + +function resolveDefault(demo: Demo, ruleName: string, paramName: string): string { + const value = + demo.perRuleParamDefaults?.[ruleName]?.[paramName] ?? demo.params[paramName]?.default ?? ""; + return typeof value === "function" ? value() : value; +} + +export default function RuleCard({ rule, demo }: Readonly<{ rule: RuleJson; demo: Demo }>) { + const initialValues = useMemo(() => { + const out: Record = {}; + for (const p of rule.params) out[p.name] = resolveDefault(demo, rule.name, p.name); + return out; + }, [rule, demo]); + + const [values, setValues] = useState>(initialValues); + const [fieldErrors, setFieldErrors] = useState>({}); + const [running, setRunning] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + const controllerRef = useRef(null); + + const nonRunnableReason = demo.nonRunnable?.[rule.name]; + + // Validation happens on Run only — it never blocks typing. + function validate(): boolean { + const next: Record = {}; + for (const p of rule.params) { + const cfg = demo.params[p.name]; + const value = values[p.name] ?? ""; + if (value.length === 0) { + next[p.name] = "required"; + } else if (value.length > MAX_PARAM_CHARS) { + next[p.name] = `at most ${MAX_PARAM_CHARS} characters`; + } else if (cfg?.pattern && !cfg.pattern.test(value)) { + next[p.name] = cfg.help ? `expected ${cfg.help}` : `must match ${cfg.pattern.source}`; + } + } + setFieldErrors(next); + return Object.keys(next).length === 0; + } + + async function run() { + if (!validate()) return; + const controller = new AbortController(); + controllerRef.current = controller; + setRunning(true); + setError(null); + setResult(null); + try { + setResult(await runQuery(rule.sql, values, controller.signal)); + } catch (e) { + if (controller.signal.aborted) { + setError(null); // user hit Cancel + } else { + const message = e instanceof Error ? e.message : String(e); + setError(message.slice(0, MAX_ERROR_CHARS)); + } + } finally { + if (controllerRef.current === controller) controllerRef.current = null; + setRunning(false); + } + } + + return ( +
+
+ + {rule.name} + + {rule.severity !== null && ( + + )} + {rule.tags.map((tag) => ( + + ))} +
+ {rule.description !== null && ( + + {rule.description} + + )} + + {rule.form} + + + + {rule.sql_pretty} + + + {nonRunnableReason ? ( +
+ +
+ ) : ( + <> + {rule.params.length > 0 && ( +
+ {rule.params.map((p) => ( + { + setValues((prev) => ({ ...prev, [p.name]: inputValue })); + setFieldErrors((prev) => { + if (!(p.name in prev)) return prev; + const rest = { ...prev }; + delete rest[p.name]; + return rest; + }); + }} + /> + ))} +
+ )} +
+ + {running && ( + + )} +
+ {error !== null && } + {result !== null && } + + )} +
+ ); +} diff --git a/site/src/demos/github-threats/main.tsx b/site/src/demos/github-threats/main.tsx new file mode 100644 index 0000000..274f46e --- /dev/null +++ b/site/src/demos/github-threats/main.tsx @@ -0,0 +1,16 @@ +import React from "react"; +import ReactDOM from "react-dom/client"; +import { ClickUIProvider } from "@clickhouse/click-ui"; +import DemoLayout from "../DemoLayout"; +import { DEMOS } from "../registry"; +import type { RulesJson } from "../types"; +import data from "../../generated/github-threats.json"; +import "../../styles.css"; + +ReactDOM.createRoot(document.getElementById("root")!).render( + + + + + +); diff --git a/site/src/demos/hackernews/main.tsx b/site/src/demos/hackernews/main.tsx new file mode 100644 index 0000000..4cfc2e9 --- /dev/null +++ b/site/src/demos/hackernews/main.tsx @@ -0,0 +1,16 @@ +import React from "react"; +import ReactDOM from "react-dom/client"; +import { ClickUIProvider } from "@clickhouse/click-ui"; +import DemoLayout from "../DemoLayout"; +import { DEMOS } from "../registry"; +import type { RulesJson } from "../types"; +import data from "../../generated/hackernews.json"; +import "../../styles.css"; + +ReactDOM.createRoot(document.getElementById("root")!).render( + + + + + +); diff --git a/site/src/demos/playground.ts b/site/src/demos/playground.ts new file mode 100644 index 0000000..a59b01f --- /dev/null +++ b/site/src/demos/playground.ts @@ -0,0 +1,115 @@ +// The only networking module on the demo pages. +// +// SECURITY CONTRACT +// - The SQL sent here comes verbatim from build-time generated JSON, compiled +// by clicklisp from this repo's own .lisp files. It is trusted, static +// content — never concatenate anything into it. +// - User input reaches the server ONLY as typed param_NAME bindings resolved +// server-side against {name:Type} placeholders, so it can never change the +// query's structure. Param names are re-validated here (defense in depth) +// even though they too come from generated JSON. +// - The server side is bounded by the readonly `demo` playground user +// (max_result_rows=1000, max_execution_time=120). +// - CRITICAL: no `headers` option on fetch. The default text/plain body is +// CORS-safelisted (no preflight); the endpoint's Access-Control-Allow-Headers +// does NOT include content-type, so setting one breaks the request. + +const PLAYGROUND = "https://sql-clickhouse.clickhouse.com/"; + +const PARAM_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; + +const TIMEOUT_MS = 65000; +const MAX_ERROR_CHARS = 2000; + +export interface RunSummary { + readRows?: number; + readBytes?: number; + elapsedNs?: number; +} + +export interface RunResult { + meta: Array<{ name: string }>; + data: unknown[][]; + rows: number; + summary?: RunSummary; +} + +function truncate(s: string, max: number): string { + return s.length > max ? `${s.slice(0, max)}…` : s; +} + +function parseSummary(header: string | null): RunSummary | undefined { + if (!header) return undefined; + try { + // JSON with string numbers, e.g. {"read_rows":"1234","elapsed_ns":"56789",...} + const raw = JSON.parse(header) as Record; + const num = (key: string): number | undefined => { + const n = Number(raw[key]); + return Number.isFinite(n) ? n : undefined; + }; + return { readRows: num("read_rows"), readBytes: num("read_bytes"), elapsedNs: num("elapsed_ns") }; + } catch { + return undefined; + } +} + +// AbortSignal.any is newer (Safari 17.4+) than the build's browser floor; +// fall back to forwarding both aborts through a fresh controller. +function combineSignals(signal: AbortSignal | undefined, timeout: AbortSignal): AbortSignal { + if (!signal) return timeout; + if (typeof AbortSignal.any === "function") return AbortSignal.any([signal, timeout]); + const controller = new AbortController(); + const forward = () => controller.abort(); + signal.addEventListener("abort", forward, { once: true }); + timeout.addEventListener("abort", forward, { once: true }); + return controller.signal; +} + +export async function runQuery( + sql: string, + params: Record, + signal?: AbortSignal +): Promise { + const url = new URL(PLAYGROUND); + url.searchParams.set("user", "demo"); + url.searchParams.set("default_format", "JSONCompact"); + for (const [name, value] of Object.entries(params)) { + if (!PARAM_NAME.test(name)) throw new Error(`invalid query parameter name: ${name}`); + url.searchParams.set(`param_${name}`, value); + } + + const timeout = AbortSignal.timeout(TIMEOUT_MS); + const res = await fetch(url, { + method: "POST", + body: sql, + signal: combineSignals(signal, timeout), + }); + + const summary = parseSummary(res.headers.get("x-clickhouse-summary")); + + const text = await res.text(); + if (!res.ok) { + throw new Error(truncate(text || `HTTP ${res.status}`, MAX_ERROR_CHARS)); + } + + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error("unexpected response shape (not JSON)"); + } + + const obj = parsed as { meta?: unknown; data?: unknown; rows?: unknown }; + const metaOk = + Array.isArray(obj.meta) && + obj.meta.every( + (m) => typeof m === "object" && m !== null && typeof (m as { name?: unknown }).name === "string" + ); + const dataOk = Array.isArray(obj.data) && obj.data.every((row) => Array.isArray(row)); + if (!metaOk || !dataOk) throw new Error("unexpected response shape"); + + const meta = obj.meta as Array<{ name: string }>; + const data = obj.data as unknown[][]; + const rows = typeof obj.rows === "number" ? obj.rows : data.length; + return { meta, data, rows, summary }; +} diff --git a/site/src/demos/registry.ts b/site/src/demos/registry.ts new file mode 100644 index 0000000..39bbcfd --- /dev/null +++ b/site/src/demos/registry.ts @@ -0,0 +1,110 @@ +import dayjs from "dayjs"; +import type { IconName } from "@clickhouse/click-ui"; + +// Everything the demo pages and the landing #demos section know about each +// demo that is NOT in the generated JSON: display copy, dataset links, and +// per-parameter UI config (defaults, validation, help). + +export interface ParamConfig { + label: string; + default: string | (() => string); + pattern?: RegExp; + help?: string; +} + +export interface Demo { + slug: string; + title: string; + blurb: string; + icon: IconName; + dataset: string; + datasetHref: string; + exampleFiles: string[]; + loadCommand: string; + params: Record; + perRuleParamDefaults?: Record string)>>; + /** Rule name → short reason it cannot run against the public playground. */ + nonRunnable?: Record; +} + +const DATETIME = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/; +const UINT32 = /^\d{1,9}$/; +const DATETIME_HELP = "YYYY-MM-DD HH:MM:SS"; + +const EXAMPLES = "https://github.com/ClickHouse/clicklisp/blob/main/examples/analytics"; + +export type DemoSlug = "github-threats" | "uk-price-paid" | "hackernews" | "repo-health"; + +export const DEMOS: Record = { + "github-threats": { + slug: "github-threats", + title: "GitHub events threat pack", + blurb: + "Detection rules hunting branch-reset storms, mass ref deletion, tag retargeting and star storms across 11B public GitHub events.", + icon: "secure", + dataset: "github.events — 11B GitHub events", + datasetHref: "https://clickhouse.com/docs/getting-started/example-datasets/github-events", + exampleFiles: [`${EXAMPLES}/github-events.lisp`], + loadCommand: + "bin/clicklisp rules sql --load examples/analytics/github-events.lisp tag-retarget | scripts/play.sh since=2025-09-01T00:00:00 until=2025-09-08T00:00:00", + params: { + since: { label: "since (DateTime)", default: "2025-01-01 00:00:00", pattern: DATETIME, help: DATETIME_HELP }, + until: { label: "until (DateTime)", default: "2025-01-08 00:00:00", pattern: DATETIME, help: DATETIME_HELP }, + hours: { label: "hours (UInt32)", default: "24", pattern: UINT32 }, + repo: { label: "repo (String)", default: "ClickHouse/ClickHouse", help: "owner/name" }, + }, + perRuleParamDefaults: { + "tag-retarget": { since: "2025-09-01 00:00:00", until: "2025-09-08 00:00:00" }, + "commit-hour-anomaly": { + since: () => dayjs().subtract(30, "day").format("YYYY-MM-DD HH:mm:ss"), + }, + }, + }, + "uk-price-paid": { + slug: "uk-price-paid", + title: "UK property prices", + blurb: "Macro-stamped league tables and price trends over 30M UK property transactions.", + icon: "home", + dataset: "uk.uk_price_paid — 30M UK property transactions", + datasetHref: "https://clickhouse.com/docs/getting-started/example-datasets/uk-price-paid", + exampleFiles: [`${EXAMPLES}/uk-price-paid.lisp`], + loadCommand: + "bin/clicklisp rules sql --load examples/analytics/uk-price-paid.lisp price-trend | scripts/play.sh town=LONDON", + params: { + town: { label: "town (String)", default: "LONDON", help: "uppercase, as stored (LONDON, YORK, ...)" }, + }, + }, + hackernews: { + slug: "hackernews", + title: "Hacker News analytics", + blurb: "Hype cycles, vocabulary and prolific authors over 49M Hacker News rows.", + icon: "fire", + dataset: "hackernews.hackernews — 49M Hacker News rows", + datasetHref: "https://sql.clickhouse.com", + exampleFiles: [`${EXAMPLES}/hackernews.lisp`], + loadCommand: + "bin/clicklisp rules sql --load examples/analytics/hackernews.lisp hype-tracker | scripts/play.sh term=clojure", + params: { + term: { label: "term (String)", default: "clojure" }, + }, + nonRunnable: { + "shouting-titles": "needs the local clicklisp_shout executable UDF", + }, + }, + "repo-health": { + slug: "repo-health", + title: "Repo health", + blurb: "Engineering analytics over ClickHouse's own git history.", + icon: "git-merge", + dataset: "git.clickhouse_* — ClickHouse's own git history", + datasetHref: "https://sql.clickhouse.com", + exampleFiles: [`${EXAMPLES}/repo-health.lisp`, `${EXAMPLES}/repo-health-playground.lisp`], + loadCommand: + "bin/clicklisp rules sql --load examples/analytics/repo-health.lisp --load examples/analytics/repo-health-playground.lisp file-rewrites | scripts/play.sh min_lines=200", + params: { + min_lines: { label: "min_lines (UInt32)", default: "200", pattern: UINT32 }, + }, + }, +}; + +export const DEMO_ORDER: DemoSlug[] = ["github-threats", "uk-price-paid", "hackernews", "repo-health"]; diff --git a/site/src/demos/repo-health/main.tsx b/site/src/demos/repo-health/main.tsx new file mode 100644 index 0000000..e65ac65 --- /dev/null +++ b/site/src/demos/repo-health/main.tsx @@ -0,0 +1,16 @@ +import React from "react"; +import ReactDOM from "react-dom/client"; +import { ClickUIProvider } from "@clickhouse/click-ui"; +import DemoLayout from "../DemoLayout"; +import { DEMOS } from "../registry"; +import type { RulesJson } from "../types"; +import data from "../../generated/repo-health.json"; +import "../../styles.css"; + +ReactDOM.createRoot(document.getElementById("root")!).render( + + + + + +); diff --git a/site/src/demos/types.ts b/site/src/demos/types.ts new file mode 100644 index 0000000..fde0a03 --- /dev/null +++ b/site/src/demos/types.ts @@ -0,0 +1,23 @@ +// Shape of the build-time JSON in src/generated/.json, emitted by the +// clicklisp binary (`make site-data` at the repo root). + +export interface RuleParam { + name: string; + type: string; +} + +export interface RuleJson { + name: string; + description: string | null; + severity: "info" | "low" | "medium" | "high" | "critical" | null; + tags: string[]; + params: RuleParam[]; + sql: string; + sql_pretty: string; + form: string; +} + +export interface RulesJson { + clicklisp: string; + rules: RuleJson[]; +} diff --git a/site/src/demos/uk-price-paid/main.tsx b/site/src/demos/uk-price-paid/main.tsx new file mode 100644 index 0000000..541d8c9 --- /dev/null +++ b/site/src/demos/uk-price-paid/main.tsx @@ -0,0 +1,16 @@ +import React from "react"; +import ReactDOM from "react-dom/client"; +import { ClickUIProvider } from "@clickhouse/click-ui"; +import DemoLayout from "../DemoLayout"; +import { DEMOS } from "../registry"; +import type { RulesJson } from "../types"; +import data from "../../generated/uk-price-paid.json"; +import "../../styles.css"; + +ReactDOM.createRoot(document.getElementById("root")!).render( + + + + + +); diff --git a/site/src/samples.ts b/site/src/samples.ts index 7b81085..6902c17 100644 --- a/site/src/samples.ts +++ b/site/src/samples.ts @@ -25,7 +25,8 @@ export const ruleExample = `;; rules.lisp export const rulesCli = `clicklisp rules --load rules.lisp # list clicklisp rules sql --load rules.lisp --all # emit SQL for all -clicklisp rules sql --load rules.lisp ssh-bruteforce`; +clicklisp rules sql --load rules.lisp ssh-bruteforce +clicklisp rules json --load rules.lisp --all # machine-readable JSON`; export const macroExample = `(defmacro def-price-league (ptype) \`(defquery ,(intern (format nil "TOP-~A-DISTRICTS" ptype)) diff --git a/site/src/sections/Demos.tsx b/site/src/sections/Demos.tsx new file mode 100644 index 0000000..f0fb6c3 --- /dev/null +++ b/site/src/sections/Demos.tsx @@ -0,0 +1,40 @@ +import { CardSecondary, GridContainer, Text, Title } from "@clickhouse/click-ui"; +import { Eyebrow, Section } from "../components"; +import { DEMOS, DEMO_ORDER } from "../demos/registry"; + +export default function Demos() { + return ( +
+
+ Live demos + + Rule packs running on real data + + + Four query libraries compiled by clicklisp at build time, runnable in your browser against + the public ClickHouse playground. + +
+ + {DEMO_ORDER.map((slug) => { + const demo = DEMOS[slug]; + return ( + + ); + })} + +
+ ); +} diff --git a/site/src/sections/Header.tsx b/site/src/sections/Header.tsx index e74ae91..93c355f 100644 --- a/site/src/sections/Header.tsx +++ b/site/src/sections/Header.tsx @@ -6,6 +6,7 @@ const NAV: Array<[string, string]> = [ ["Rules", "#rules"], ["UDFs", "#udfs"], ["Playground", "#playground"], + ["Demos", "#demos"], ["FAQ", "#faq"], ]; diff --git a/site/src/styles.css b/site/src/styles.css index 634dc22..19ee0b9 100644 --- a/site/src/styles.css +++ b/site/src/styles.css @@ -518,6 +518,90 @@ code { justify-content: space-between; } +/* ---------- demo pages ---------- */ + +.demo-page .section-inner { + max-width: 980px; +} + +.demo-lede { + display: flex; + flex-direction: column; + gap: 8px; + max-width: 720px; +} + +.demo-load { + display: flex; + flex-direction: column; + gap: 10px; + margin-top: 26px; +} + +.rule-list { + display: flex; + flex-direction: column; + gap: 26px; + margin-top: 34px; +} + +.rule-card { + display: flex; + flex-direction: column; + gap: 14px; + border: 1px solid var(--border-soft); + border-radius: 8px; + background: var(--surface); + padding: 22px; +} + +.rule-card-head { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +.rule-card-head .rule-name { + margin: 0 8px 0 0 !important; + font-family: "Inconsolata", ui-monospace, Menlo, Consolas, monospace !important; + font-size: 1.15rem !important; +} + +.param-row { + display: flex; + gap: 14px; + flex-wrap: wrap; + align-items: flex-start; +} + +.param-row > * { + flex: 1 1 220px; + min-width: 0; + max-width: 340px; +} + +.run-row { + display: flex; + gap: 10px; + align-items: center; + flex-wrap: wrap; +} + +.stats-row { + display: flex; + gap: 8px; + flex-wrap: wrap; +} + +.results { + display: flex; + flex-direction: column; + gap: 10px; + min-width: 0; + overflow-x: auto; +} + /* ---------- misc ---------- */ .centered { diff --git a/site/tsconfig.json b/site/tsconfig.json index 390f55c..1bc4f6b 100644 --- a/site/tsconfig.json +++ b/site/tsconfig.json @@ -7,6 +7,7 @@ "skipLibCheck": true, "moduleResolution": "bundler", "allowImportingTsExtensions": true, + "resolveJsonModule": true, "isolatedModules": true, "moduleDetection": "force", "types": ["vite/client"], diff --git a/site/uk-price-paid/index.html b/site/uk-price-paid/index.html new file mode 100644 index 0000000..434a1b3 --- /dev/null +++ b/site/uk-price-paid/index.html @@ -0,0 +1,24 @@ + + + + + + + + + clicklisp demos — UK property prices + + + + + +
+ + + diff --git a/site/vite.config.ts b/site/vite.config.ts index 5c00060..bb181c1 100644 --- a/site/vite.config.ts +++ b/site/vite.config.ts @@ -1,11 +1,47 @@ -import { defineConfig } from "vite"; +import { resolve } from "node:path"; +import { defineConfig, type Plugin } from "vite"; import react from "@vitejs/plugin-react"; +// Defense-in-depth: GitHub Pages cannot set real response headers, so the +// Content-Security-Policy ships as a tag baked into every page at +// build time. The policy is centralized here — this is the only place it is +// defined. Build-only on purpose: dev mode needs Vite's inline react-refresh +// preamble, which this policy would block. +const CSP = + "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src https://fonts.gstatic.com; img-src 'self' data:; connect-src https://sql-clickhouse.clickhouse.com https://api.github.com; base-uri 'none'; form-action 'none'; object-src 'none'; frame-src 'none'; manifest-src 'self'"; + +function cspPlugin(): Plugin { + return { + name: "csp-meta", + apply: "build", + transformIndexHtml() { + return [ + { + tag: "meta", + attrs: { "http-equiv": "Content-Security-Policy", content: CSP }, + injectTo: "head-prepend", + }, + ]; + }, + }; +} + // Deployed to GitHub Pages behind the custom domain https://cl.clickhouse.com/ // (served from the root — if the custom domain is ever removed, the site moves // back under https://clickhouse.github.io/clicklisp/ and base must become // "/clicklisp/" again). export default defineConfig({ base: "/", - plugins: [react()], + plugins: [react(), cspPlugin()], + build: { + rollupOptions: { + input: { + main: resolve(import.meta.dirname, "index.html"), + "github-threats": resolve(import.meta.dirname, "github-threats/index.html"), + "uk-price-paid": resolve(import.meta.dirname, "uk-price-paid/index.html"), + hackernews: resolve(import.meta.dirname, "hackernews/index.html"), + "repo-health": resolve(import.meta.dirname, "repo-health/index.html"), + }, + }, + }, }); diff --git a/src/main.lisp b/src/main.lisp index 4f2045d..71f5ebf 100644 --- a/src/main.lisp +++ b/src/main.lisp @@ -84,6 +84,136 @@ (format t "~A~%" (first compiled))))) 0)) +;;; --------------------------------------------------------------------- +;;; clicklisp rules json: machine-readable rule dumps + +(defun json-string-literal (string) + "Render STRING as a double-quoted JSON string literal. The output is +pure ASCII (control characters and everything past ~ escape to \\uXXXX, +astral code points as UTF-16 surrogate pairs), which makes it immune to +stream external-format differences." + (with-output-to-string (out) + (flet ((escape (code) + (format out "\\u~(~4,'0x~)" code))) + (write-char #\" out) + (loop for char across string + for code = (char-code char) + do (cond ((char= char #\") (write-string "\\\"" out)) + ((char= char #\\) (write-string "\\\\" out)) + ((char= char #\Newline) (write-string "\\n" out)) + ((char= char #\Tab) (write-string "\\t" out)) + ((char= char #\Return) (write-string "\\r" out)) + ((< code 32) (escape code)) + ((> code #xFFFF) + (let ((offset (- code #x10000))) + (escape (+ #xD800 (ash offset -10))) + (escape (+ #xDC00 (logand offset #x3FF))))) + ((> code 126) (escape code)) + (t (write-char char out)))) + (write-char #\" out)))) + +(defun rule-params (rule) + "The {name:Type} query parameters in RULE's query, as a list of +\(NAME . TYPE) rendered strings, deduplicated by name in first-appearance +order. Parameters spliced in through (raw \"...\") strings are not seen." + (let ((params '())) + (labels ((param-form-p (form) + (and (head-name-p form "PARAM") + (consp (rest form)) + (consp (cddr form)) + (null (cdddr form)))) + (record (name type) + ;; render exactly as the compiler's param special form does + (let ((rendered (etypecase name + (string name) + (symbol (substitute #\_ #\- (symbol-sql-name name)))))) + (unless (assoc rendered params :test #'string=) + (push (cons rendered (type-sql type)) params)))) + (walk (form) + (cond ((param-form-p form) + (record (second form) (third form))) + ((consp form) + (walk (car form)) + (walk (cdr form))) + ;; vector literals compile element-wise, so params + ;; can hide inside them (strings hold no forms) + ((and (vectorp form) (not (stringp form))) + (map nil #'walk form))))) + (walk (rule-query rule))) + (nreverse params))) + +(defun rule-definition-form (rule) + "Rebuild the DEFRULE form (DEFQUERY when severity-free) that would +define RULE, as data." + (let ((severity (rule-severity rule)) + (options '())) + (when (rule-tags rule) + (setf options (list* :tags (rule-tags rule) options))) + (when severity + (setf options (list* :severity severity options))) + (when (rule-description rule) + (setf options (list* :description (rule-description rule) options))) + (list (if severity 'defrule 'defquery) + (intern (string-upcase (rule-name rule)) '#:clicklisp.forms) + options + (rule-query rule)))) + +(defun rule-form-text (rule) + "RULE's definition form, pretty-printed the way rule files write it." + (let ((*package* (find-package '#:clicklisp.forms)) + (*print-pretty* t) + (*print-case* :downcase) + (*print-right-margin* 78) + (*print-level* nil) + (*print-length* nil) + (*print-readably* nil)) + (write-to-string (rule-definition-form rule)))) + +(defun print-rule-json (rule stream) + (let ((severity (rule-severity rule))) + (format stream " {~%") + (format stream " \"name\": ~A,~%" + (json-string-literal (rule-name rule))) + (format stream " \"description\": ~A,~%" + (if (rule-description rule) + (json-string-literal (rule-description rule)) + "null")) + (format stream " \"severity\": ~A,~%" + (if severity + (json-string-literal (string-downcase (symbol-name severity))) + "null")) + (format stream " \"tags\": [~{~A~^, ~}],~%" + (mapcar (lambda (tag) + (json-string-literal (string-downcase (symbol-name tag)))) + (rule-tags rule))) + (format stream " \"params\": [~{~A~^, ~}],~%" + (mapcar (lambda (param) + (format nil "{\"name\": ~A, \"type\": ~A}" + (json-string-literal (car param)) + (json-string-literal (cdr param)))) + (rule-params rule))) + (format stream " \"sql\": ~A,~%" + (json-string-literal (rule-sql rule))) + (format stream " \"sql_pretty\": ~A,~%" + (json-string-literal (rule-sql rule :pretty t))) + (format stream " \"form\": ~A~%" + (json-string-literal (rule-form-text rule))) + (format stream " }"))) + +(defun print-rules-json (rules &optional (stream *standard-output*)) + "Emit RULES as one JSON object carrying both compact and pretty SQL." + (format stream "{~%") + (format stream " \"clicklisp\": ~A,~%" (json-string-literal *version*)) + (cond ((null rules) + (format stream " \"rules\": []~%")) + (t + (format stream " \"rules\": [~%") + (loop for (rule . more) on rules + do (print-rule-json rule stream) + (format stream "~:[~;,~]~%" more)) + (format stream " ]~%"))) + (format stream "}~%")) + ;;; --------------------------------------------------------------------- ;;; clicklisp rules @@ -108,7 +238,7 @@ ;; the subcommand is the first positional wherever it appears, so ;; `rules --load f.lisp sql --all` works like `rules sql --load f.lisp --all` (let ((subcommand (if (and positionals - (member (first positionals) '("list" "sql") + (member (first positionals) '("list" "sql" "json") :test #'string=)) (pop positionals) "list")) @@ -141,7 +271,19 @@ (let ((severity (rule-severity rule))) (and severity (string-downcase (symbol-name severity)))) (rule-description rule) - (rule-sql rule :pretty pretty))))))) + (rule-sql rule :pretty pretty))))) + ((string= subcommand "json") + ;; --pretty is accepted but a no-op here: the JSON always carries + ;; both compact and pretty SQL. + (let ((rules (cond (all (list-rules)) + (names (mapcar (lambda (name) + (or (find-rule name) + (usage-error "rules: unknown rule ~A" name))) + names)) + (t (usage-error "rules json: pass rule names or --all"))))) + (when (null rules) + (usage-error "rules json: no rules loaded (use --load FILE)")) + (print-rules-json rules))))) 0)) ;;; --------------------------------------------------------------------- @@ -240,6 +382,9 @@ commands:~%~ rules [list] [--load FILE] list loaded rules and queries~%~ rules sql [--load FILE] [--pretty] [--all | NAME ...]~%~ emit SQL for rules and queries~%~ + rules json [--load FILE] [--all | NAME ...]~%~ + machine-readable JSON (with both~%~ + compact and pretty SQL)~%~ udf --fn NAME [--load FILE] [--chunked] [--watch FILE]~%~ serve a UDF over stdin/stdout~%~ (TabSeparated, for ClickHouse~%~ diff --git a/tests/rules-json-tests.lisp b/tests/rules-json-tests.lisp new file mode 100644 index 0000000..206f8e7 --- /dev/null +++ b/tests/rules-json-tests.lisp @@ -0,0 +1,84 @@ +;;;; Tests for the machine-readable `rules json` output. + +(in-package #:clicklisp/test) + +(deftest json-string-escaping + (is= "\"\"" (clicklisp::json-string-literal "")) + (is= "\"plain ascii 123\"" (clicklisp::json-string-literal "plain ascii 123")) + (is= "\"a\\\"b\"" (clicklisp::json-string-literal "a\"b")) + (is= "\"a\\\\b\"" (clicklisp::json-string-literal "a\\b")) + (is= "\"a\\nb\"" (clicklisp::json-string-literal (format nil "a~%b"))) + (is= "\"a\\tb\"" (clicklisp::json-string-literal (format nil "a~Cb" #\Tab))) + (is= "\"a\\rb\"" (clicklisp::json-string-literal (format nil "a~Cb" #\Return))) + ;; control characters below space escape to \u00XX (lowercase hex) + (is= "\"\\u0001\"" (clicklisp::json-string-literal (string (code-char 1)))) + ;; non-ASCII escapes too, so the output is external-format-proof + (is= "\"caf\\u00e9\"" (clicklisp::json-string-literal + (format nil "caf~C" (code-char #xE9)))) + ;; astral code points become UTF-16 surrogate pairs + (when (> char-code-limit #x10000) + (is= "\"\\ud83d\\ude00\"" + (clicklisp::json-string-literal (string (code-char #x1F600)))))) + +(defrule json-params-rule + (:description "params fixture" :severity :low :tags (:test)) + (select (a) + :from t1 + :where (and (> a (param min-lines :uint32)) + (< a (param min-lines :uint32)) + (= town (param "town" :string)) + (> ts (param since :date-time))))) + +(defrule json-vector-params-rule + (:severity :low) + (select (x) :from t1 :where (in x #((param lo :uint32) (param hi :uint32))))) + +(deftest rule-params-extraction + ;; dedupe by rendered name, first-appearance order, compiler renderings: + ;; kebab->snake for symbols, strings verbatim, :date-time -> DateTime + (is= '(("min_lines" . "UInt32") ("town" . "String") ("since" . "DateTime")) + (clicklisp::rule-params (find-rule "json-params-rule"))) + (is= '() (clicklisp::rule-params (find-rule "test-rule"))) + ;; vector literals compile element-wise, so the walk must enter them + (is= '(("lo" . "UInt32") ("hi" . "UInt32")) + (clicklisp::rule-params (find-rule "json-vector-params-rule")))) + +(defrule json-full-rule + (:description "json fixture" :severity :high :tags (:t1110 :ssh)) + (select ((as (count) n)) :from logins :where (= ok (param flag :uint8)))) + +(deftest rules-json-emission + (let ((out (with-output-to-string (s) + (clicklisp::print-rules-json + (list (find-rule "json-full-rule")) s)))) + (is (search (format nil "\"clicklisp\": \"~A\"" clicklisp:*version*) out)) + (is (search "\"name\": \"json-full-rule\"" out)) + (is (search "\"description\": \"json fixture\"" out)) + (is (search "\"severity\": \"high\"" out)) + (is (search "\"tags\": [\"t1110\", \"ssh\"]" out)) + (is (search "\"params\": [{\"name\": \"flag\", \"type\": \"UInt8\"}]" out)) + (is (search "\"sql\": \"SELECT count() AS n FROM logins WHERE (ok = {flag:UInt8})\"" out)) + (is (search "\"sql_pretty\": \"SELECT count() AS n\\nFROM logins\\nWHERE (ok = {flag:UInt8})\"" out)) + (is (search "\"form\": \"(defrule json-full-rule" out)))) + +(defquery json-bare-query () (select (1))) + +(deftest rules-json-nulls + (let ((out (with-output-to-string (s) + (clicklisp::print-rules-json + (list (find-rule "json-bare-query")) s)))) + (is (search "\"description\": null" out)) + (is (search "\"severity\": null" out)) + (is (search "\"tags\": []" out)) + (is (search "\"params\": []" out)) + (is (search "\"sql\": \"SELECT 1\"" out)))) + +(deftest rules-json-form-round-trip + (dolist (name '("json-full-rule" "json-bare-query" "json-params-rule")) + (let* ((rule (find-rule name)) + (form (let ((*read-eval* nil) + (*package* (find-package '#:clicklisp.forms))) + (read-from-string (clicklisp::rule-form-text rule))))) + (is (member (first form) '(defrule defquery))) + (is= (string-upcase (rule-name rule)) (symbol-name (second form))) + (is= (rule-sql rule) (compile-query (fourth form))))))