Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/code-quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ jobs:

steps:
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: Setup Node.js
uses: actions/setup-node@v4
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22.x"
cache: "npm"

- name: Install dependencies
run: npm ci
run: npx --yes npm@11.15.0 ci

- name: Check code formatting
run: npm run format:check
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/ping-qdrant.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,16 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: Setup Node.js
uses: actions/setup-node@v4
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22"
cache: "npm"

- name: Install dependencies
run: npm ci
run: npx --yes npm@11.15.0 ci

- name: Ping Qdrant cluster
env:
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,23 @@ jobs:

steps:
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: Setup Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: ${{ matrix.node-version }}
cache: "npm"

- name: Install dependencies
run: npm ci
run: npx --yes npm@11.15.0 ci

- name: Run tests with coverage
run: npm run test:coverage
if: matrix.node-version == '24.x'

- name: Upload coverage reports
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
if: matrix.node-version == '24.x'
with:
name: coverage-report
Expand Down
103 changes: 58 additions & 45 deletions .github/workflows/update-vector-store.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,16 @@ on:
jobs:
update-vector-store:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read

# Only run on successful production deployments (or manual trigger)
if: >
github.event_name == 'workflow_dispatch' ||
(
github.event_name == 'workflow_dispatch' &&
github.ref == 'refs/heads/main'
) ||
(
github.event.deployment_status.state == 'success' &&
github.event.deployment.environment == 'Production'
Expand All @@ -30,7 +36,7 @@ jobs:
- name: Check PR labels
if: github.event_name != 'workflow_dispatch'
id: pr-check
uses: actions/github-script@v7
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
with:
script: |
const sha = context.payload.deployment.sha;
Expand Down Expand Up @@ -75,74 +81,79 @@ jobs:
# ── Checkout & detect changed dictionary files ─────────────────────
- name: Checkout repository
if: github.event_name == 'workflow_dispatch' || steps.pr-check.outputs.should_continue == 'true'
uses: actions/checkout@v4
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.deployment.sha || github.sha }}
fetch-depth: 2

- name: Detect changed dictionary files
if: github.event_name == 'workflow_dispatch' || steps.pr-check.outputs.should_continue == 'true'
id: detect
env:
MANUAL_UPSERT_SLUGS: ${{ github.event.inputs.upsert_slugs }}
MANUAL_DELETE_SLUGS: ${{ github.event.inputs.delete_slugs }}
run: |
# For manual triggers, use the provided inputs directly
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
UPSERT="${{ github.event.inputs.upsert_slugs }}"
DELETE="${{ github.event.inputs.delete_slugs }}"

if [ -z "$UPSERT" ] && [ -z "$DELETE" ]; then
echo "has_changes=false" >> "$GITHUB_OUTPUT"
echo "No slugs provided for manual trigger."
else
echo "upsert=$UPSERT" >> "$GITHUB_OUTPUT"
echo "delete=$DELETE" >> "$GITHUB_OUTPUT"
echo "has_changes=true" >> "$GITHUB_OUTPUT"
validate_slug_list() {
local slug_list="$1"

if [ -z "$slug_list" ] || [[ "$slug_list" =~ ^[a-z0-9]+(-[a-z0-9]+)*(,[a-z0-9]+(-[a-z0-9]+)*)*$ ]]; then
return 0
fi
exit 0
fi

# For deployment triggers, diff against the parent commit
echo "Detecting dictionary file changes..."
echo "::error::Slugs must be lowercase letters and digits separated by single hyphens."
return 1
}

UPSERT_SLUGS=""
DELETE_SLUGS=""

while IFS=$'\t' read -r status file; do
if [[ "$file" == src/content/dictionary/*.mdx ]]; then
slug=$(basename "$file" .mdx)

if [[ "$status" == "D" ]]; then
DELETE_SLUGS="${DELETE_SLUGS:+$DELETE_SLUGS,}$slug"
else
UPSERT_SLUGS="${UPSERT_SLUGS:+$UPSERT_SLUGS,}$slug"
if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then
UPSERT_SLUGS="$MANUAL_UPSERT_SLUGS"
DELETE_SLUGS="$MANUAL_DELETE_SLUGS"
else
echo "Detecting dictionary file changes..."
while IFS=$'\t' read -r status file; do
if [[ "$file" == src/content/dictionary/*.mdx ]]; then
slug=$(basename "$file" .mdx)

if [[ "$status" == "D" ]]; then
DELETE_SLUGS="${DELETE_SLUGS:+$DELETE_SLUGS,}$slug"
else
UPSERT_SLUGS="${UPSERT_SLUGS:+$UPSERT_SLUGS,}$slug"
fi
fi
fi
done < <(git diff --name-status HEAD~1 -- src/content/dictionary/)
done < <(git diff --name-status HEAD~1 -- src/content/dictionary/)
fi

if ! validate_slug_list "$UPSERT_SLUGS" || ! validate_slug_list "$DELETE_SLUGS"; then
exit 1
fi

printf 'upsert=%s\n' "$UPSERT_SLUGS" >> "$GITHUB_OUTPUT"
printf 'delete=%s\n' "$DELETE_SLUGS" >> "$GITHUB_OUTPUT"

if [ -z "$UPSERT_SLUGS" ] && [ -z "$DELETE_SLUGS" ]; then
echo "has_changes=false" >> "$GITHUB_OUTPUT"
printf 'has_changes=false\n' >> "$GITHUB_OUTPUT"
echo "No dictionary file changes detected. Skipping update."
else
echo "upsert=$UPSERT_SLUGS" >> "$GITHUB_OUTPUT"
echo "delete=$DELETE_SLUGS" >> "$GITHUB_OUTPUT"
echo "has_changes=true" >> "$GITHUB_OUTPUT"
echo "Upsert slugs: $UPSERT_SLUGS"
echo "Delete slugs: $DELETE_SLUGS"
printf 'has_changes=true\n' >> "$GITHUB_OUTPUT"
echo "Dictionary changes detected."
fi

- name: Skip — no dictionary changes
if: steps.detect.outputs.has_changes != 'true' && (github.event_name == 'workflow_dispatch' || steps.pr-check.outputs.should_continue == 'true')
run: echo "⏭️ No dictionary changes to process. Skipping."

# ── Run the incremental update ─────────────────────────────────────
- name: Setup Node.js
if: steps.detect.outputs.has_changes == 'true'
uses: actions/setup-node@v4
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22"
cache: "npm"

- name: Install dependencies
if: steps.detect.outputs.has_changes == 'true'
run: npm ci
run: npx --yes npm@11.15.0 ci

- name: Update vector store
if: steps.detect.outputs.has_changes == 'true'
Expand All @@ -151,19 +162,21 @@ jobs:
OPENAI_EMBEDDINGS_MODEL: ${{ vars.OPENAI_EMBEDDINGS_MODEL }}
QDRANT_URL: ${{ secrets.QDRANT_URL }}
QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}
UPSERT_SLUGS: ${{ steps.detect.outputs.upsert }}
DELETE_SLUGS: ${{ steps.detect.outputs.delete }}
run: |
ARGS=""
ARGS=()

if [ -n "${{ steps.detect.outputs.upsert }}" ]; then
ARGS="$ARGS --upsert ${{ steps.detect.outputs.upsert }}"
if [ -n "$UPSERT_SLUGS" ]; then
ARGS+=(--upsert "$UPSERT_SLUGS")
fi

if [ -n "${{ steps.detect.outputs.delete }}" ]; then
ARGS="$ARGS --delete ${{ steps.detect.outputs.delete }}"
if [ -n "$DELETE_SLUGS" ]; then
ARGS+=(--delete "$DELETE_SLUGS")
fi

echo "Running: npm run update:jai:ci -- $ARGS"
npm run update:jai:ci -- $ARGS
echo "Running incremental vector store update."
npm run update:jai:ci -- "${ARGS[@]}"

- name: Update successful
if: steps.detect.outputs.has_changes == 'true'
Expand Down
4 changes: 2 additions & 2 deletions dev/seed-vector-store.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import fetch from "node-fetch";
import { Document } from "langchain/document";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { Document } from "@langchain/classic/document";
import { RecursiveCharacterTextSplitter } from "@langchain/classic/text_splitter";
import { vectorStore } from "../apps/jai/index.js";

// Load the dictionary from the jargons.dev API
Expand Down
32 changes: 18 additions & 14 deletions dev/update-vector-store.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,25 @@
*/

import fetch from "node-fetch";
import { Document } from "langchain/document";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { Document } from "@langchain/classic/document";
import { RecursiveCharacterTextSplitter } from "@langchain/classic/text_splitter";
import { vectorStore } from "../apps/jai/index.js";

const PRODUCTION_API_BASE = "https://jargons.dev/api/v1/browse";
const SLUG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;

function parseSlugList(value, option) {
const slugs = value.split(",").map((slug) => slug.trim());

if (slugs.some((slug) => !SLUG_PATTERN.test(slug))) {
console.error(
`❌ Error: ${option} slugs must be lowercase letters and digits separated by single hyphens.`,
);
process.exit(1);
}

return slugs;
}

// ---------------------------------------------------------------------------
// CLI argument parsing
Expand All @@ -44,12 +58,7 @@ function parseArgs(argv) {
);
process.exit(1);
}
upsertSlugs.push(
...args[i + 1]
.split(",")
.map((s) => s.trim())
.filter(Boolean),
);
upsertSlugs.push(...parseSlugList(args[i + 1], "--upsert"));
i++;
} else if (args[i] === "--delete") {
if (!args[i + 1] || args[i + 1].startsWith("--")) {
Expand All @@ -61,12 +70,7 @@ function parseArgs(argv) {
);
process.exit(1);
}
deleteSlugs.push(
...args[i + 1]
.split(",")
.map((s) => s.trim())
.filter(Boolean),
);
deleteSlugs.push(...parseSlugList(args[i + 1], "--delete"));
i++;
} else {
console.error(`❌ Error: Unknown argument "${args[i]}".`);
Expand Down
Loading