diff --git a/.github/workflows/custom.yaml b/.github/workflows/custom.yaml index bdd02fe..1780f0e 100644 --- a/.github/workflows/custom.yaml +++ b/.github/workflows/custom.yaml @@ -43,8 +43,11 @@ jobs: - name: Run local GitHub Action uses: ./ + with: + # TODO: support pg 16 explicitly + postgres-version: 17 env: GITHUB_TOKEN: ${{ github.token }} - POSTGRES_URL: postgres://query_doctor@localhost:5432/testing + SOURCE_DATABASE_URL: postgres://query_doctor@localhost:5432/testing LOG_PATH: /var/log/postgresql/postgres.log SITE_API_ENDPOINT: ${{ vars.SITE_API_ENDPOINT }} diff --git a/action.yaml b/action.yaml index 84893a2..5e5010f 100644 --- a/action.yaml +++ b/action.yaml @@ -8,6 +8,12 @@ branding: icon: "database" color: "blue" +inputs: + postgres-version: + description: "PostgreSQL version to use (14, 17, or 18)" + required: true + default: "18" + runs: using: "composite" steps: @@ -51,13 +57,34 @@ runs: sudo make install # Use sudo to install globally cd ${{ github.action_path }} # Return to action directory + - name: Start PostgreSQL + shell: bash + run: | + PORT=$(shuf -i 10000-65000 -n 1) + echo "PG_PORT=$PORT" >> $GITHUB_ENV + sudo apt-get install -y curl ca-certificates + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/postgresql.gpg + echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt-get update + sudo apt-get install -y postgresql-client-18 || sudo apt-get install -y postgresql-client-17 + PG_CLIENT_VERSION=$(ls /usr/lib/postgresql | sort -rn | head -1) + echo "PG_DUMP_BINARY=/usr/lib/postgresql/${PG_CLIENT_VERSION}/bin/pg_dump" >> $GITHUB_ENV + echo "PG_RESTORE_BINARY=/usr/lib/postgresql/${PG_CLIENT_VERSION}/bin/pg_restore" >> $GITHUB_ENV + docker run -d \ + --name query-doctor-postgres \ + -p $PORT:5432 \ + ghcr.io/query-doctor/postgres:pg-${{ inputs.postgres-version }} + until docker exec query-doctor-postgres pg_isready -U postgres; do + sleep 1 + done + # Run the application - name: Run Analyzer shell: bash working-directory: ${{ github.action_path }} run: npm run start env: - PG_DUMP_BINARY: /usr/bin/pg_dump CI: "true" GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} SITE_API_ENDPOINT: ${{ env.SITE_API_ENDPOINT }} + POSTGRES_URL: postgresql://postgres@localhost:${{ env.PG_PORT }}/postgres diff --git a/src/main.ts b/src/main.ts index 7e032c8..176acc9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,9 +15,9 @@ import { formatCost, queryPreview } from "./reporters/github/github.ts"; import { DEFAULT_CONFIG, fetchAnalyzerConfig } from "./config.ts"; async function runInCI( - postgresUrl: Connectable, + targetPostgresUrl: Connectable, + sourcePostgresUrl: Connectable, logPath: string, - statisticsPath?: string, maxCost?: number, ) { const siteApiEndpoint = env.SITE_API_ENDPOINT; @@ -31,8 +31,8 @@ async function runInCI( : DEFAULT_CONFIG; const runner = await Runner.build({ - postgresUrl, - statisticsPath, + targetPostgresUrl, + sourcePostgresUrl, logPath, maxCost, ignoredQueryHashes: config.ignoredQueryHashes, @@ -85,8 +85,8 @@ async function runInCI( log.info( "main", `No baseline found on branch "${comparisonBranch}". Comparison will be skipped. ` + - `To establish a baseline, run the analyzer on pushes to "${comparisonBranch}" ` + - `(add "push: branches: [${comparisonBranch}]" to your workflow trigger).`, + `To establish a baseline, run the analyzer on pushes to "${comparisonBranch}" ` + + `(add "push: branches: [${comparisonBranch}]" to your workflow trigger).`, ); } } @@ -100,6 +100,7 @@ async function runInCI( ); } + console.log("Creating report...") // Generate PR comment with comparison data await runner.report(reportContext); @@ -153,14 +154,18 @@ async function main() { core.setFailed("POSTGRES_URL environment variable is not set"); process.exit(1); } + if (!env.SOURCE_DATABASE_URL) { + core.setFailed("SOURCE_DATABASE_URL environment variable is not set"); + process.exit(1); + } if (!env.LOG_PATH) { core.setFailed("LOG_PATH environment variable is not set"); process.exit(1); } await runInCI( Connectable.fromString(env.POSTGRES_URL), + Connectable.fromString(env.SOURCE_DATABASE_URL), env.LOG_PATH, - env.STATISTICS_PATH, typeof env.MAX_COST === "number" ? env.MAX_COST : undefined, ); } else { @@ -168,4 +173,9 @@ async function main() { } } -await main(); +try { + await main(); +} catch (error) { + console.error(error); + process.exit(1); +} diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index daabacf..061e5e9 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -100,6 +100,10 @@ export class QueryOptimizer extends EventEmitter { return this._finish.promise; } + get statisticsMode(): StatisticsMode { + return this.target?.statistics.mode ?? QueryOptimizer.defaultStatistics; + } + getDisabledIndexes(): PgIdentifier[] { return [...this.disabledIndexes]; } diff --git a/src/remote/remote.ts b/src/remote/remote.ts index 69892ba..425dd2f 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -71,6 +71,7 @@ export class Remote extends EventEmitter { /** The manager for ONLY the source db connections */ private readonly sourceManager: ConnectionManager = ConnectionManager .forRemoteDatabase(), + private readonly options: { disableQueryLoader: boolean } = { disableQueryLoader: false } ) { super(); this.baseDbURL = targetURL.withDatabaseName(Remote.baseDbName); @@ -108,6 +109,10 @@ export class Remote extends EventEmitter { this.getDatabaseInfo(source), ]); + if (restoreResult.status === "rejected") { + throw new Error(`Schema sync failed: ${restoreResult.reason}`); + } + if (fullSchema.status === "fulfilled") { this.schemaLoader?.update(fullSchema.value); } @@ -200,7 +205,7 @@ export class Remote extends EventEmitter { * there isn't already an in-flight request */ private async pollQueriesOnce() { - if (this.queryLoader && !this.isPolling) { + if (this.queryLoader && !this.isPolling && !this.options.disableQueryLoader) { try { this.isPolling = true; await this.queryLoader.poll(); @@ -373,7 +378,9 @@ export class Remote extends EventEmitter { log.error("Query loader exited", "remote"); this.queryLoader = undefined; }); - this.queryLoader.start(); + if (!this.options.disableQueryLoader) { + this.queryLoader.start(); + } } async cleanup(): Promise { diff --git a/src/reporters/site-api.ts b/src/reporters/site-api.ts index 7ac4bb3..94f404f 100644 --- a/src/reporters/site-api.ts +++ b/src/reporters/site-api.ts @@ -1,7 +1,7 @@ import * as github from "@actions/github"; import type { IndexRecommendation, Nudge, SQLCommenterTag, TableReference } from "@query-doctor/core"; import { DEFAULT_CONFIG, type AnalyzerConfig } from "../config.ts"; -import type { QueryProcessResult } from "../runner.ts"; +import type { OptimizedQuery } from "../sql/recent-query.ts"; interface CiRunPayload { repo: string; @@ -25,25 +25,25 @@ export interface CiQueryPayload { export type CiOptimization = | { - state: "improvements_available"; - cost: number; - optimizedCost: number; - costReductionPercentage: number; - indexRecommendations: CiIndexRecommendation[]; - indexesUsed: string[]; - explainPlan?: object; - optimizedExplainPlan?: object; - } + state: "improvements_available"; + cost: number; + optimizedCost: number; + costReductionPercentage: number; + indexRecommendations: CiIndexRecommendation[]; + indexesUsed: string[]; + explainPlan?: object; + optimizedExplainPlan?: object; + } | { - state: "no_improvement_found"; - cost: number; - indexesUsed: string[]; - explainPlan?: object; - } + state: "no_improvement_found"; + cost: number; + indexesUsed: string[]; + explainPlan?: object; + } | { - state: "error"; - error: string; - }; + state: "error"; + error: string; + }; interface CiIndexRecommendation { schema: string; @@ -114,105 +114,25 @@ function mapIndexRecommendation(rec: IndexRecommendation): CiIndexRecommendation }; } -function mapResultToQuery(result: QueryProcessResult): CiQueryPayload | null { - switch (result.kind) { - case "recommendation": - return { - hash: result.recommendation.fingerprint, - query: result.rawQuery, - formattedQuery: result.recommendation.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: { - state: "improvements_available", - cost: result.recommendation.baseCost, - optimizedCost: result.recommendation.optimizedCost, - costReductionPercentage: - result.recommendation.baseCost > 0 - ? ((result.recommendation.baseCost - result.recommendation.optimizedCost) / - result.recommendation.baseCost) * - 100 - : 0, - indexRecommendations: result.indexRecommendations.map(mapIndexRecommendation), - indexesUsed: result.recommendation.existingIndexes, - explainPlan: result.recommendation.baseExplainPlan, - optimizedExplainPlan: result.recommendation.explainPlan, - }, - }; - - case "no_improvement": - return { - hash: result.fingerprint, - query: result.rawQuery, - formattedQuery: result.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: { - state: "no_improvement_found", - cost: result.cost, - indexesUsed: result.existingIndexes, - explainPlan: result.explainPlan, - }, - }; - - case "zero_cost_plan": - return { - hash: result.fingerprint, - query: result.rawQuery, - formattedQuery: result.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: { - state: "no_improvement_found", - cost: 0, - indexesUsed: [], - explainPlan: result.explainPlan, - }, - }; - - case "error": - return { - hash: result.fingerprint, - query: result.rawQuery, - formattedQuery: result.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: { - state: "error", - error: result.error.message, - }, - }; - - case "cost_past_threshold": - return { - hash: result.warning.fingerprint, - query: result.rawQuery, - formattedQuery: result.warning.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: result.warning.optimization - ? { - state: "no_improvement_found", - cost: result.warning.baseCost, - indexesUsed: result.warning.optimization.existingIndexes, - explainPlan: result.warning.explainPlan, - } - : { - state: "no_improvement_found", - cost: result.warning.baseCost, - indexesUsed: [], - explainPlan: result.warning.explainPlan, - }, - }; - - case "invalid": - return null; +function mapResultToQuery(result: OptimizedQuery): CiQueryPayload | null { + const { optimization } = result; + if ( + optimization.state === "waiting" || + optimization.state === "optimizing" || + optimization.state === "not_supported" || + optimization.state === "timeout" + ) { + return null; } + return { + hash: result.hash, + query: result.query, + formattedQuery: result.formattedQuery, + nudges: result.nudges, + tags: result.tags, + tableReferences: result.tableReferences ?? [], + optimization, + }; } function getQueryCost(q: CiQueryPayload): number | null { @@ -228,7 +148,7 @@ function getQueryIndexes(q: CiQueryPayload): string[] { } export function buildQueries( - results: QueryProcessResult[], + results: OptimizedQuery[], config: AnalyzerConfig = DEFAULT_CONFIG, ): CiQueryPayload[] { const ignoredSet = new Set(config.ignoredQueryHashes); diff --git a/src/runner.ts b/src/runner.ts index 111c31c..959ee8f 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -1,27 +1,9 @@ import * as core from "@actions/core"; -import * as prettier from "prettier"; -import prettierPluginSql from "prettier-plugin-sql"; import csv from "fast-csv"; -import { Readable } from "node:stream"; -import { statSync, readFileSync } from "node:fs"; +import { statSync } from "node:fs"; import { spawn } from "node:child_process"; import { fingerprint } from "@libpg-query/parser"; import { preprocessEncodedJson } from "./sql/json.ts"; -import { - Analyzer, - ExportedStats, - IndexedTable, - IndexOptimizer, - type IndexRecommendation, - type Nudge, - type SQLCommenterTag, - type TableReference, - OptimizeResult, - type Postgres, - PostgresQueryBuilder, - Statistics, - StatisticsMode, -} from "@query-doctor/core"; import { ExplainedLog } from "./sql/pg_log.ts"; import { GithubReporter } from "./reporters/github/github.ts"; import { @@ -29,52 +11,43 @@ import { type ReportContext, type ReportIndexRecommendation, type ReportQueryCostWarning, - type ReportStatistics, } from "./reporters/reporter.ts"; import { DEFAULT_CONFIG, type AnalyzerConfig } from "./config.ts"; -const bgBrightMagenta = (s: string) => `\x1b[105m${s}\x1b[0m`; -const yellow = (s: string) => `\x1b[33m${s}\x1b[0m`; -const blue = (s: string) => `\x1b[34m${s}\x1b[0m`; import { env } from "./env.ts"; -import { connectToSource } from "./sql/postgresjs.ts"; -import { parse } from "@libpg-query/parser"; import { Connectable } from "./sync/connectable.ts"; +import { Remote } from "./remote/remote.ts"; +import { ConnectionManager } from "./sync/connection-manager.ts"; +import { RecentQuery } from "./sql/recent-query.ts"; +import { QueryHash } from "./sql/recent-query.ts"; +import type { OptimizedQuery } from "./sql/recent-query.ts"; export class Runner { - private readonly seenQueries = new Set(); - public readonly queryStats: ReportStatistics = { - total: 0, - errored: 0, - matched: 0, - optimized: 0, - }; constructor( - private readonly db: Postgres, - private readonly optimizer: IndexOptimizer, - private readonly existingIndexes: IndexedTable[], - private readonly stats: Statistics, + private readonly remote: Remote, private readonly logPath: string, private readonly maxCost?: number, private readonly ignoredQueryHashes: Set = new Set(), ) { } static async build(options: { - postgresUrl: Connectable; + targetPostgresUrl: Connectable; + sourcePostgresUrl: Connectable; statisticsPath?: string; maxCost?: number; logPath: string; ignoredQueryHashes?: string[]; }) { - const db = connectToSource(options.postgresUrl); - const statisticsMode = Runner.decideStatisticsMode(options.statisticsPath); - const stats = await Statistics.fromPostgres(db, statisticsMode); - const existingIndexes = await stats.getExistingIndexes(); - const optimizer = new IndexOptimizer(db, stats, existingIndexes); + const remote = new Remote( + options.targetPostgresUrl, + ConnectionManager.forLocalDatabase(), + ConnectionManager.forRemoteDatabase(), + // queries are already sourced from logs + { disableQueryLoader: true } + ); + await remote.syncFrom(options.sourcePostgresUrl); + await remote.optimizer.finish; return new Runner( - db, - optimizer, - existingIndexes, - stats, + remote, options.logPath, options.maxCost, new Set(options.ignoredQueryHashes ?? []), @@ -82,7 +55,7 @@ export class Runner { } async close() { - await (this.db as unknown as { close(): Promise }).close(); + await this.remote.cleanup(); } async run(config: AnalyzerConfig = DEFAULT_CONFIG) { @@ -107,14 +80,14 @@ export class Runner { headers: false, }) .on("error", (err) => { + console.error("Got a pgbadger error", err); error = err; }); - const recommendations: ReportIndexRecommendation[] = []; - const queriesPastThreshold: ReportQueryCostWarning[] = []; - const allResults: QueryProcessResult[] = []; + let total = 0; console.time("total"); + const recentQueries: RecentQuery[] = []; for await (const chunk of stream) { const [ _timestamp, @@ -135,6 +108,7 @@ export class Runner { if (loglevel !== "LOG" || !queryString.startsWith("plan:")) { continue; } + total++; const planString: string = queryString.split("plan:")[1].trim(); const json = preprocessEncodedJson(planString); if (!json) { @@ -151,30 +125,66 @@ export class Runner { ); continue; } - const result = await this.processQuery(parsed); - if (result.kind !== "invalid") { - allResults.push(result); + + const query = parsed.query; + const hash = QueryHash.parse(await fingerprint(query)); + if (this.ignoredQueryHashes.has(hash)) { + continue; } - switch (result.kind) { - case "error": - this.queryStats.errored++; - break; - case "cost_past_threshold": - queriesPastThreshold.push(result.warning); - break; - case "recommendation": - recommendations.push(result.recommendation); - break; - case "no_improvement": - case "zero_cost_plan": - case "invalid": - break; + if (parsed.isIntrospection) { + continue; } + + const recentQuery = await RecentQuery.fromLogEntry(query, hash); + recentQueries.push(recentQuery) } - await new Promise((resolve) => child.on("close", () => resolve())); + console.log("Finished pgbadger stream"); + await this.remote.optimizer.addQueries(recentQueries); + + await this.remote.optimizer.finish; + + const optimizedQueries = this.remote.optimizer.getQueries(); + console.log( - `Matched ${this.queryStats.matched} queries out of ${this.queryStats.total}`, + `Matched ${this.remote.optimizer.validQueriesProcessed} queries out of ${total}`, ); + + const recommendations: ReportIndexRecommendation[] = []; + const queriesPastThreshold: ReportQueryCostWarning[] = []; + const allResults: OptimizedQuery[] = []; + + for (const q of optimizedQueries) { + if (this.ignoredQueryHashes.has(q.hash)) { + continue; + } + allResults.push(q); + const { optimization } = q; + if (optimization.state === "improvements_available") { + recommendations.push({ + fingerprint: q.hash, + formattedQuery: q.formattedQuery, + baseCost: optimization.cost, + baseExplainPlan: optimization.explainPlan, + optimizedCost: optimization.optimizedCost, + existingIndexes: optimization.indexesUsed, + proposedIndexes: optimization.indexRecommendations.map((r) => r.definition), + explainPlan: optimization.optimizedExplainPlan, + }); + } else if ( + optimization.state === "no_improvement_found" && + typeof this.maxCost === "number" && + optimization.cost > this.maxCost + ) { + queriesPastThreshold.push({ + fingerprint: q.hash, + formattedQuery: q.formattedQuery, + baseCost: optimization.cost, + explainPlan: optimization.explainPlan, + maxCost: this.maxCost, + }); + } + } + const filteredRecommendations = config.minimumCost > 0 ? recommendations.filter((r) => r.baseCost > config.minimumCost) @@ -183,12 +193,10 @@ export class Runner { config.minimumCost > 0 ? queriesPastThreshold.filter((w) => w.baseCost > config.minimumCost) : queriesPastThreshold; - const statistics = deriveIndexStatistics(filteredRecommendations); - const timeElapsed = Date.now() - startDate.getTime(); + if (config.minimumCost > 0) { const filtered = - recommendations.length - - filteredRecommendations.length + + recommendations.length - filteredRecommendations.length + (queriesPastThreshold.length - filteredThresholdWarnings.length); if (filtered > 0) { console.log( @@ -196,11 +204,19 @@ export class Runner { ); } } + + const statistics = deriveIndexStatistics(filteredRecommendations); + const timeElapsed = Date.now() - startDate.getTime(); const reportContext: ReportContext = { - statisticsMode: this.stats.mode, + statisticsMode: this.remote.optimizer.statisticsMode, recommendations: filteredRecommendations, queriesPastThreshold: filteredThresholdWarnings, - queryStats: Object.freeze(this.queryStats), + queryStats: Object.freeze({ + total, + matched: this.remote.optimizer.validQueriesProcessed, + optimized: filteredRecommendations.length, + errored: optimizedQueries.filter((q) => q.optimization.state === "error").length, + }), statistics, error, metadata: { logSize, timeElapsed }, @@ -214,317 +230,6 @@ export class Runner { console.log(`Generating report (${reporter.provider()})`); await reporter.report(reportContext); } - - async processQuery(log: ExplainedLog): Promise { - this.queryStats.total++; - const { query } = log; - const queryFingerprint = await fingerprint(query); - if (this.ignoredQueryHashes.has(queryFingerprint)) { - if (env.DEBUG) { - console.log("Skipping ignored query", queryFingerprint); - } - return { kind: "invalid" }; - } - if (log.isIntrospection) { - if (env.DEBUG) { - console.log("Skipping introspection query", queryFingerprint); - } - return { kind: "invalid" }; - } - if (this.seenQueries.has(queryFingerprint)) { - if (env.DEBUG) { - console.log("Skipping duplicate query", queryFingerprint); - } - return { kind: "invalid" }; - } - this.seenQueries.add(queryFingerprint); - - const analyzer = new Analyzer(parse); - const formattedQuery = await this.formatQuery(query); - const { indexesToCheck, ansiHighlightedQuery, referencedTables, nudges, tags } = - await analyzer.analyze(formattedQuery); - - const selectsCatalog = referencedTables.find((ref) => - ref.table.startsWith("pg_"), - ); - if (selectsCatalog) { - if (env.DEBUG) { - console.log( - "Skipping query that selects from catalog tables", - selectsCatalog, - queryFingerprint, - ); - } - return { kind: "invalid" }; - } - const indexCandidates = analyzer.deriveIndexes( - this.stats.ownMetadata, - indexesToCheck, - referencedTables, - ); - if (indexCandidates.length === 0) { - if (env.DEBUG) { - console.log(ansiHighlightedQuery); - console.log("No index candidates found", queryFingerprint); - } - if (typeof this.maxCost === "number" && log.plan.cost > this.maxCost) { - return { - kind: "cost_past_threshold", - rawQuery: query, - nudges, - tags, - referencedTables, - warning: { - fingerprint: queryFingerprint, - formattedQuery, - baseCost: log.plan.cost, - explainPlan: log.plan.json, - maxCost: this.maxCost, - }, - }; - } - } - return core.group( - `query:${queryFingerprint}`, - async (): Promise => { - console.time(`timing`); - this.printLegend(); - console.log(ansiHighlightedQuery); - // TODO: give concrete type - let out: OptimizeResult; - this.queryStats.matched++; - try { - const builder = new PostgresQueryBuilder(query); - out = await this.optimizer.run(builder, indexCandidates); - } catch (err) { - console.error(err); - console.error( - `Something went wrong while running this query. Skipping`, - ); - // this.queryStats.errored++; - console.timeEnd(`timing`); - return { - kind: "error", - error: err as Error, - fingerprint: queryFingerprint, - rawQuery: query, - formattedQuery, - nudges, - tags, - referencedTables, - }; - } - if (out.kind === "ok") { - const existingIndexesForQuery = Array.from(out.existingIndexes) - .map((index) => { - const existing = this.existingIndexes.find( - (e) => e.index_name === index, - ); - if (existing) { - return `${existing.schema_name}.${existing.table_name}(${existing.index_columns - .map((c) => `"${c.name}" ${c.order}`) - .join(", ")})`; - } - }) - .filter((i) => i !== undefined); - if (out.newIndexes.size > 0) { - const costReductionPct = out.baseCost > 0 - ? ((out.baseCost - out.finalCost) / out.baseCost) * 100 - : 0; - if (Math.round(costReductionPct) <= 0) { - console.log( - `Skipping recommendation with ${costReductionPct.toFixed(1)}% cost reduction (rounds to 0%)`, - ); - console.timeEnd(`timing`); - return { - kind: "no_improvement", - fingerprint: queryFingerprint, - rawQuery: query, - formattedQuery, - cost: out.baseCost, - existingIndexes: existingIndexesForQuery, - nudges, - tags, - referencedTables, - explainPlan: out.baseExplainPlan, - }; - } - this.queryStats.optimized++; - const newIndexRecommendations = Array.from(out.newIndexes) - .map((n) => out.triedIndexes.get(n)) - .filter((n) => n !== undefined); - const newIndexes = newIndexRecommendations.map((n) => n.definition); - console.log(`New indexes: ${newIndexes.join(", ")}`); - return { - kind: "recommendation", - rawQuery: query, - nudges, - tags, - referencedTables, - indexRecommendations: newIndexRecommendations, - recommendation: { - fingerprint: queryFingerprint, - formattedQuery, - baseCost: out.baseCost, - baseExplainPlan: out.baseExplainPlan, - optimizedCost: out.finalCost, - existingIndexes: existingIndexesForQuery, - proposedIndexes: newIndexes, - explainPlan: out.explainPlan, - }, - }; - } else { - console.log("No new indexes found"); - if ( - typeof this.maxCost === "number" && - out.finalCost > this.maxCost - ) { - console.log( - "Query cost is too high", - out.finalCost, - this.maxCost, - ); - return { - kind: "cost_past_threshold", - rawQuery: query, - nudges, - tags, - referencedTables, - warning: { - fingerprint: queryFingerprint, - formattedQuery, - baseCost: out.baseCost, - optimization: { - newCost: out.finalCost, - existingIndexes: existingIndexesForQuery, - proposedIndexes: [], - }, - explainPlan: out.explainPlan, - maxCost: this.maxCost, - }, - }; - } - return { - kind: "no_improvement", - fingerprint: queryFingerprint, - rawQuery: query, - formattedQuery, - cost: out.baseCost, - existingIndexes: existingIndexesForQuery, - nudges, - tags, - referencedTables, - explainPlan: out.baseExplainPlan, - }; - } - } else if (out.kind === "zero_cost_plan") { - console.log("Zero cost plan found"); - console.log(out); - console.timeEnd(`timing`); - return { - kind: "zero_cost_plan", - explainPlan: out.explainPlan, - fingerprint: queryFingerprint, - rawQuery: query, - formattedQuery, - nudges, - tags, - referencedTables, - }; - } - console.timeEnd(`timing`); - console.error(out); - throw new Error(`Unexpected output: ${out}`); - }, - ); - } - - private async formatQuery(query: string): Promise { - try { - return await prettier.format(query, { - parser: "sql", - plugins: [prettierPluginSql], - language: "postgresql", - keywordCase: "upper", - }); - } catch { - return query; - } - } - - private printLegend() { - console.log(`--Legend--------------------------`); - console.log(`| ${bgBrightMagenta(" column ")} | Candidate |`); - console.log(`| ${yellow(" column ")} | Ignored |`); - console.log(`| ${blue(" column ")} | Temp table reference |`); - console.log(`-----------------------------------`); - console.log(); - } - - private static decideStatisticsMode(path?: string): StatisticsMode { - if (path) { - const data = Runner.readStatisticsFile(path); - return Statistics.statsModeFromExport(data); - } else { - return Statistics.defaultStatsMode; - } - } - private static readStatisticsFile(path: string): ExportedStats[] { - const data = readFileSync(path); - const json = JSON.parse(new TextDecoder().decode(data)); - return ExportedStats.array().parse(json); - } } -export type QueryProcessResult = - | { - kind: "invalid"; - } - | { - kind: "cost_past_threshold"; - rawQuery: string; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - warning: ReportQueryCostWarning; - } - | { - kind: "recommendation"; - rawQuery: string; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - indexRecommendations: IndexRecommendation[]; - recommendation: ReportIndexRecommendation; - } - | { - kind: "no_improvement"; - fingerprint: string; - rawQuery: string; - formattedQuery: string; - cost: number; - existingIndexes: string[]; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - explainPlan?: object; - } | { - kind: "error"; - error: Error; - fingerprint: string; - rawQuery: string; - formattedQuery: string; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - } - | { - kind: "zero_cost_plan"; - explainPlan: object; - fingerprint: string; - rawQuery: string; - formattedQuery: string; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - }; +export type QueryProcessResult = OptimizedQuery; diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index 35d5d55..ede7d9f 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -75,6 +75,22 @@ export class RecentQuery { */ private static readonly MAX_ANALYZABLE_QUERY_SIZE = 50_000; + static fromLogEntry(query: string, hash: QueryHash, seenAt: number = Date.now()) { + return RecentQuery.analyze( + { + query, + formattedQuery: query, + username: "", + meanTime: 0, + calls: "1", + rows: "0", + topLevel: true, + }, + hash, + seenAt, + ); + } + static async analyze( data: RawRecentQuery, hash: QueryHash,