diff --git a/.gitignore b/.gitignore index 397a63fb2..fa7de171b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ dist/ releases/ +.devspace-dev/ .env *.log diff --git a/README.md b/README.md index 3caa568bf..fed1d2049 100644 --- a/README.md +++ b/README.md @@ -253,9 +253,16 @@ Install pnpm 11.25.0, the version pinned in `package.json`, with ```bash pnpm install --frozen-lockfile +pnpm dev:seed pnpm dev pnpm typecheck pnpm test pnpm build pnpm start ``` + +`dev:seed` forks your normal DevSpace config and SQLite state into an ignored +checkout-local `.devspace-dev/` directory so source builds and migrations do not +modify your normal installation. Use `pnpm dev:reset` to discard that QA state +and fork it again. See [Development and Manual QA](docs/development.md) for +worktree switching, ChatGPT, and database-migration workflows. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 000000000..ce55db2eb --- /dev/null +++ b/docs/development.md @@ -0,0 +1,97 @@ +# Development and Manual QA + +Use the published DevSpace installation for normal work. When testing DevSpace +itself, run the source checkout against a checkout-local fork of your DevSpace +configuration and SQLite state. + +## First run in a checkout + +Install dependencies, then seed the checkout from your normal DevSpace setup: + +```bash +pnpm install --frozen-lockfile +pnpm dev:seed +pnpm dev +``` + +`dev:seed` creates an ignored `.devspace-dev/` directory in the current +checkout. It copies the current config, auth file, DevSpace-local skills and +agent profiles, and makes a SQLite backup of the configured state database. The +copied config is rewritten so `storage.stateDir` points at the checkout-local +state directory. + +`pnpm dev` only uses that local QA configuration. If the checkout has not been +seeded, it stops with an instruction to run `pnpm dev:seed` instead of silently +falling back to your normal DevSpace state. + +By default the seed source is `~/.devspace`. If your normal installation uses a +custom `DEVSPACE_CONFIG_DIR`, keep that value exported while using `dev:seed` +and `dev:reset` so both commands fork the same installation. + +## Testing with ChatGPT + +Stop the installed DevSpace server before starting the source checkout so both +processes do not compete for the configured port. You can keep the same tunnel +and public URL running. + +Because the QA database is forked from your normal state, it starts with the +same registered OAuth clients and current access and refresh tokens. This +usually lets ChatGPT continue through a server restart without setting up a new +connection. + +The fork is a snapshot, not shared state. OAuth refresh tokens rotate when they +are used, so a long-lived QA fork can diverge from the normal installation or +from another worktree's older fork. Do not rely on separate QA databases to +remain permanently interchangeable without re-authentication. + +## Switching between worktrees + +Each worktree keeps its own `.devspace-dev/` state: + +```bash +# worktree A +pnpm dev:seed +pnpm dev + +# stop it, then switch to worktree B +pnpm dev:seed +pnpm dev +``` + +Once a worktree has been seeded, later runs only need `pnpm dev`. + +This keeps source changes and persistent QA state isolated without requiring +DevSpace to know which Git branch or worktree is active. + +## Database and migration changes + +Do not point experimental source builds at your normal DevSpace state directory. +Use the checkout-local fork so migrations operate on disposable data that began +as a realistic copy of your current installation. + +To repeat a migration from the same baseline, discard the checkout QA state and +fork it again: + +```bash +pnpm dev:reset +pnpm dev +``` + +`dev:reset` replaces the entire `.devspace-dev/` directory from the current +normal DevSpace config and state. Any QA-only workspace sessions, OAuth changes, +agent sessions, and database migrations in that checkout are discarded. + +DevSpace also validates the migration journal at startup. If an applied +migration version has a different name than the current build expects, or the +database contains a migration version unknown to the build, startup fails +instead of silently using an incompatible schema. + +## Normal verification + +The usual repository checks remain: + +```bash +pnpm typecheck +pnpm test +pnpm build +``` diff --git a/docs/setup.md b/docs/setup.md index 427bea5dc..12ae05b0c 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -154,7 +154,10 @@ pinned in `package.json`. Install it with `npm install --global pnpm@11.25.0`. ```bash pnpm install --frozen-lockfile +pnpm dev:seed pnpm dev ``` -The same setup rules apply. +The source server uses an ignored checkout-local fork of your normal DevSpace +configuration and SQLite state. See [Development and Manual QA](development.md) +for worktree switching, ChatGPT testing, and database migration workflows. diff --git a/package.json b/package.json index 3a09cfdfe..fb6fe5903 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,9 @@ "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", "build": "pnpm clean && pnpm build:app && tsc -p tsconfig.build.json", "build:app": "vite build", - "dev": "tsx watch --clear-screen=false src/cli.ts serve", + "dev": "tsx scripts/dev.ts", + "dev:seed": "tsx scripts/dev-state.ts seed", + "dev:reset": "tsx scripts/dev-state.ts reset", "postinstall": "node scripts/fix-node-pty-permissions.mjs", "prepack": "pnpm build", "schema:config": "tsx scripts/generate-config-schema.ts", diff --git a/scripts/dev-state.ts b/scripts/dev-state.ts new file mode 100644 index 000000000..4dc92ebf8 --- /dev/null +++ b/scripts/dev-state.ts @@ -0,0 +1,157 @@ +import { existsSync } from "node:fs"; +import { + chmod, + cp, + mkdir, + readFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { homedir } from "node:os"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import Database from "better-sqlite3"; +import { parse, type ParseError } from "jsonc-parser"; +import { migrateLegacyConfig } from "../src/config-migration.js"; +import { devspaceConfigSchema, type DevspaceConfig } from "../src/config-schema.js"; +import { databasePath } from "../src/db/client.js"; +import { expandHomePath } from "../src/roots.js"; + +const checkoutRoot = resolve(process.cwd()); +const devRoot = join(checkoutRoot, ".devspace-dev"); + +export async function seedDevState({ reset = false }: { reset?: boolean } = {}): Promise { + const sourceConfigDir = resolve( + expandHomePath(process.env.DEVSPACE_CONFIG_DIR ?? join(homedir(), ".devspace")), + ); + if (isWithin(devRoot, sourceConfigDir)) { + throw new Error("Refusing to seed development state from this checkout's own .devspace-dev directory."); + } + + const source = await readSourceConfig(sourceConfigDir); + const sourceStateDir = resolve(expandHomePath(source.config.storage.stateDir)); + const sourceDatabasePath = databasePath(sourceStateDir); + + if (existsSync(devRoot) && !reset) { + throw new Error("Development state is already initialized. Run `pnpm dev:reset` to replace it."); + } + + const stagingRoot = `${devRoot}.staging-${process.pid}-${Date.now()}`; + const stagingConfigDir = join(stagingRoot, "config"); + const stagingStateDir = join(stagingRoot, "state"); + const devConfigDir = join(devRoot, "config"); + const devStateDir = join(devRoot, "state"); + + try { + await mkdir(stagingConfigDir, { recursive: true }); + await mkdir(stagingStateDir, { recursive: true }); + + const localConfig: DevspaceConfig = { + ...source.config, + storage: { + ...source.config.storage, + stateDir: devStateDir, + }, + }; + const localConfigPath = join(stagingConfigDir, "config.jsonc"); + await writeFile(localConfigPath, `${JSON.stringify(localConfig, null, 2)}\n`, { mode: 0o600 }); + + const sourceAuthPath = join(sourceConfigDir, "auth.json"); + if (existsSync(sourceAuthPath)) { + const localAuthPath = join(stagingConfigDir, "auth.json"); + await cp(sourceAuthPath, localAuthPath); + await chmod(localAuthPath, 0o600); + } else if (!process.env.DEVSPACE_OAUTH_OWNER_TOKEN) { + throw new Error(`No auth.json found in ${sourceConfigDir}. Run DevSpace setup before seeding development state.`); + } + + for (const directory of ["skills", "agents"] as const) { + const sourceDirectory = join(sourceConfigDir, directory); + if (existsSync(sourceDirectory)) { + await cp(sourceDirectory, join(stagingConfigDir, directory), { recursive: true }); + } + } + + if (existsSync(sourceDatabasePath)) { + await backupDatabase(sourceDatabasePath, databasePath(stagingStateDir)); + } + + await promoteStagedState(stagingRoot, reset); + } finally { + await rm(stagingRoot, { recursive: true, force: true }); + } + + console.log(`${reset ? "Reset" : "Seeded"} development state in ${devRoot}`); +} + +async function promoteStagedState(stagingRoot: string, reset: boolean): Promise { + if (!reset || !existsSync(devRoot)) { + await rename(stagingRoot, devRoot); + return; + } + + const previousRoot = `${devRoot}.previous-${process.pid}-${Date.now()}`; + await rename(devRoot, previousRoot); + try { + await rename(stagingRoot, devRoot); + } catch (error) { + await rename(previousRoot, devRoot); + throw error; + } + await rm(previousRoot, { recursive: true, force: true }); +} + +function isWithin(parent: string, candidate: string): boolean { + const pathFromParent = relative(parent, candidate); + return pathFromParent === "" + || (pathFromParent !== ".." + && !pathFromParent.startsWith(`..${sep}`) + && !isAbsolute(pathFromParent)); +} + +async function readSourceConfig(configDir: string): Promise<{ config: DevspaceConfig }> { + const configPath = join(configDir, "config.jsonc"); + if (existsSync(configPath)) { + const source = await readFile(configPath, "utf8"); + const errors: ParseError[] = []; + const value = parse(source, errors, { allowTrailingComma: true }); + if (errors.length > 0) { + throw new Error(`Unable to parse ${configPath}.`); + } + return { config: devspaceConfigSchema.parse(value) }; + } + + const legacyPath = join(configDir, "config.json"); + if (existsSync(legacyPath)) { + const value = JSON.parse(await readFile(legacyPath, "utf8")) as unknown; + return { config: migrateLegacyConfig(value) }; + } + + throw new Error(`No DevSpace configuration found in ${configDir}. Run DevSpace setup before seeding development state.`); +} + +async function backupDatabase(sourcePath: string, destinationPath: string): Promise { + await mkdir(dirname(destinationPath), { recursive: true }); + const source = new Database(sourcePath, { readonly: true, fileMustExist: true }); + try { + await source.backup(destinationPath); + await chmod(destinationPath, 0o600); + } finally { + source.close(); + } +} + +async function main(): Promise { + const command = process.argv[2]; + if (command === "seed") { + await seedDevState(); + return; + } + if (command === "reset") { + await seedDevState({ reset: true }); + return; + } + throw new Error("Usage: dev-state "); +} + +await main(); diff --git a/scripts/dev.ts b/scripts/dev.ts new file mode 100644 index 000000000..b5326531c --- /dev/null +++ b/scripts/dev.ts @@ -0,0 +1,33 @@ +import { existsSync } from "node:fs"; +import { join, resolve } from "node:path"; +import spawn from "cross-spawn"; + +const checkoutRoot = resolve(process.cwd()); +const configDir = join(checkoutRoot, ".devspace-dev", "config"); +const hasConfig = existsSync(join(configDir, "config.jsonc")) || existsSync(join(configDir, "config.json")); + +if (!hasConfig) { + console.error("Development state is not initialized. Run `pnpm dev:seed` first."); + process.exitCode = 1; +} else { + const child = spawn("tsx", ["watch", "--clear-screen=false", "src/cli.ts", "serve"], { + cwd: checkoutRoot, + env: { + ...process.env, + DEVSPACE_CONFIG_DIR: configDir, + }, + stdio: "inherit", + }); + + child.on("error", (error) => { + console.error(error); + process.exitCode = 1; + }); + child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + process.exitCode = code ?? 1; + }); +} diff --git a/src/config-migration.ts b/src/config-migration.ts index f24851adb..837d46f32 100644 --- a/src/config-migration.ts +++ b/src/config-migration.ts @@ -10,6 +10,7 @@ import { LOCAL_AGENT_PROVIDERS } from "./local-agent-profiles.js"; const legacyConfigSchema = z.object({ host: z.string().optional(), port: z.number().optional(), + tool_mode: z.enum(["claude", "codex"]).optional(), allowedRoots: z.array(z.string()).optional(), publicBaseUrl: z.string().nullable().optional(), allowedHosts: z.array(z.string()).optional(), @@ -30,6 +31,7 @@ const legacyConfigSchema = z.object({ const LEGACY_CONFIG_KEYS = new Set([ "host", "port", + "tool_mode", "allowedRoots", "publicBaseUrl", "allowedHosts", @@ -65,7 +67,7 @@ export function migrateLegacyConfig(value: unknown): DevspaceConfig { worktreeRoot: legacy.worktreeRoot, }), storage: definedEntries({ stateDir: legacy.stateDir }), - tools: definedEntries({ mode: legacy.tools?.mode }), + tools: definedEntries({ mode: legacy.tools?.mode ?? legacy.tool_mode }), ui: definedEntries({ enabled: legacy.ui?.enabled }), artifacts: definedEntries({ enabled: legacy.artifactsEnabled, diff --git a/src/db/migrations.ts b/src/db/migrations.ts index df192caa0..9a4a5611d 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -49,13 +49,24 @@ export function migrateDatabase(sqlite: Database.Database): void { ); `); - const applied = new Set( - ( - sqlite.prepare("select version from devspace_schema_migrations").all() as Array<{ - version: number; - }> - ).map((row) => row.version), - ); + const appliedRows = sqlite + .prepare("select version, name from devspace_schema_migrations order by version") + .all() as Array<{ version: number; name: string }>; + const migrationsByVersion = new Map(migrations.map((migration) => [migration.version, migration])); + for (const row of appliedRows) { + const expected = migrationsByVersion.get(row.version); + if (!expected) { + throw new Error( + `Database migration history is incompatible: version ${row.version} (${JSON.stringify(row.name)}) is unknown to this build.`, + ); + } + if (row.name !== expected.name) { + throw new Error( + `Database migration history is incompatible: version ${row.version} is recorded as ${JSON.stringify(row.name)}, but this build expects ${JSON.stringify(expected.name)}.`, + ); + } + } + const applied = new Set(appliedRows.map((row) => row.version)); const recordMigration = sqlite.prepare( "insert into devspace_schema_migrations (version, name, applied_at) values (?, ?, ?)", ); diff --git a/src/dev-state.test.ts b/src/dev-state.test.ts new file mode 100644 index 000000000..04dd8ee74 --- /dev/null +++ b/src/dev-state.test.ts @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import Database from "better-sqlite3"; + +const root = await mkdtemp(join(tmpdir(), "devspace-dev-state-test-")); +const checkoutRoot = join(root, "checkout"); +const sourceConfigDir = join(root, "config"); +const sourceStateDir = join(root, "state"); +const scriptPath = fileURLToPath(new URL("../scripts/dev-state.ts", import.meta.url)); +const tsxCliPath = fileURLToPath(import.meta.resolve("tsx/cli")); + +try { + await mkdir(checkoutRoot, { recursive: true }); + await mkdir(join(sourceConfigDir, "skills", "example"), { recursive: true }); + await mkdir(sourceStateDir, { recursive: true }); + await writeFile(join(sourceConfigDir, "config.jsonc"), JSON.stringify({ + configVersion: 1, + storage: { stateDir: sourceStateDir }, + })); + await writeFile(join(sourceConfigDir, "auth.json"), JSON.stringify({ + ownerToken: "test-owner-token-that-is-long-enough", + })); + await writeFile(join(sourceConfigDir, "skills", "example", "SKILL.md"), "example skill\n"); + + const sourceDatabase = new Database(join(sourceStateDir, "devspace.sqlite")); + sourceDatabase.exec("create table marker (value text); insert into marker values ('source')"); + sourceDatabase.close(); + + await runDevState("seed"); + + const devRoot = join(checkoutRoot, ".devspace-dev"); + const localConfig = JSON.parse( + await readFile(join(devRoot, "config", "config.jsonc"), "utf8"), + ) as { storage: { stateDir: string } }; + assert.equal( + await realpath(localConfig.storage.stateDir), + await realpath(join(devRoot, "state")), + ); + assert.equal(existsSync(join(devRoot, "config", "auth.json")), true); + assert.equal(existsSync(join(devRoot, "config", "skills", "example", "SKILL.md")), true); + + const localDatabasePath = join(devRoot, "state", "devspace.sqlite"); + const localDatabase = new Database(localDatabasePath); + assert.equal(localDatabase.prepare("select value from marker").pluck().get(), "source"); + localDatabase.exec("insert into marker values ('local-only')"); + localDatabase.close(); + + await assert.rejects(runDevState("seed"), /already initialized/); + + await rm(join(sourceConfigDir, "auth.json")); + await assert.rejects(runDevState("reset"), /No auth\.json found/); + + const preservedDatabase = new Database(localDatabasePath, { readonly: true }); + try { + assert.deepEqual( + preservedDatabase.prepare("select value from marker order by rowid").pluck().all(), + ["source", "local-only"], + ); + } finally { + preservedDatabase.close(); + } + + await writeFile(join(sourceConfigDir, "auth.json"), JSON.stringify({ + ownerToken: "test-owner-token-that-is-long-enough", + })); + await runDevState("reset"); + + const resetDatabase = new Database(localDatabasePath, { readonly: true }); + assert.deepEqual(resetDatabase.prepare("select value from marker order by rowid").pluck().all(), ["source"]); + resetDatabase.close(); +} finally { + await rm(root, { recursive: true, force: true }); +} + +console.log("dev state tests passed"); + +async function runDevState(command: "seed" | "reset"): Promise { + return new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [tsxCliPath, scriptPath, command], + { + cwd: checkoutRoot, + env: { + ...process.env, + DEVSPACE_CONFIG_DIR: sourceConfigDir, + DEVSPACE_OAUTH_OWNER_TOKEN: "", + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + let stderr = ""; + child.stderr.setEncoding("utf8").on("data", (chunk: string) => { + stderr += chunk; + }); + child.once("error", reject); + child.once("close", (code) => { + if (code === 0) resolve(); + else reject(new Error(stderr || `dev-state exited with ${code}`)); + }); + }); +} diff --git a/src/user-config.test.ts b/src/user-config.test.ts index 8e09b9122..e45eeb572 100644 --- a/src/user-config.test.ts +++ b/src/user-config.test.ts @@ -19,6 +19,7 @@ withConfigDir((configDir, env) => { writeFileSync(join(configDir, "config.json"), JSON.stringify({ host: "0.0.0.0", port: 8787, + tool_mode: "claude", allowedRoots: ["/work"], publicBaseUrl: "https://devspace.example.com", artifactsEnabled: true, @@ -35,7 +36,7 @@ withConfigDir((configDir, env) => { assert.deepEqual(files.config.workspaces.allowedRoots, ["/work"]); assert.equal(files.config.artifacts.enabled, true); assert.equal(files.config.subagents.enabled, true); - assert.equal(files.config.tools.mode, "codex"); + assert.equal(files.config.tools.mode, "claude"); assert.equal(files.config.ui.enabled, true); assert.equal(files.auth.ownerToken, "test-owner-token"); assert.equal(existsSync(join(configDir, "config.json")), false);