Fix version history

.changeset/moody-ducks-warn.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Support forked_from metadata key in openfn.yaml

integration-tests/cli/test/sync.test.ts

@@ -30,7 +30,10 @@ const initWorkspace = (t: any) => {
   };
 };
 
-const gen = (name = 'patients', workflows = ['trigger-job(body="fn()")']) => {
+const gen = (
+  name = 'patients',
+  workflows = ['trigger-job(expression="fn()")']
+) => {
   // generate a project
   const project = generateProject(name, workflows, {
     openfnUuid: true,
@@ -44,7 +47,7 @@ test('fetch a new project', async (t) => {
   const { workspace, read } = initWorkspace(t);
   const project = gen();
 
-  await run(
+  const { stdout } = await run(
     `openfn project fetch \
        --workspace ${workspace} \
        --endpoint ${endpoint} \
@@ -239,7 +242,7 @@ test('pull an update to project', async (t) => {
 test('checkout by alias', async (t) => {
   const { workspace, read } = initWorkspace(t);
   const main = gen();
-  const staging = gen('patients-staging', ['trigger-job(body="fn(x)")']);
+  const staging = gen('patients-staging', ['trigger-job(expression="fn(x)")']);
 
   await run(
     `openfn project fetch \

packages/cli/CHANGELOG.md

@@ -4,7 +4,7 @@
 
 ### Minor Changes
 
-- 8b9f402: fetch: allow state files to be writtem to JSON with --format
+- 8b9f402: fetch: allow state files to be written to JSON with --format
 
 ### Patch Changes
 

packages/cli/src/projects/checkout.ts

@@ -10,7 +10,7 @@ import * as o from '../options';
 import * as po from './options';
 
 import type { Opts } from './options';
-import { tidyWorkflowDir } from './util';
+import { tidyWorkflowDir, updateForkedFrom } from './util';
 
 export type CheckoutOptions = Pick<
   Opts,
@@ -69,7 +69,11 @@ export const handler = async (options: CheckoutOptions, logger: Logger) => {
     await tidyWorkflowDir(currentProject!, switchProject);
   }
 
+  // write the forked from map
+  updateForkedFrom(switchProject);
+
   // expand project into directory
+  // TODO: only write files with a diff
   const files: any = switchProject.serialize('fs');
   for (const f in files) {
     if (files[f]) {

packages/cli/src/projects/deploy.ts

@@ -1,6 +1,8 @@
 import yargs from 'yargs';
 import Project from '@openfn/project';
 import c from 'chalk';
+import { writeFile } from 'node:fs/promises';
+import path from 'node:path';
 
 import * as o from '../options';
 import * as o2 from './options';
@@ -10,6 +12,7 @@ import {
   fetchProject,
   serialize,
   getSerializePath,
+  updateForkedFrom,
 } from './util';
 import { build, ensure } from '../util/command-builders';
 
@@ -64,6 +67,34 @@ export const command: yargs.CommandModule<DeployOptions> = {
   handler: ensure('project-deploy', options),
 };
 
+export const hasRemoteDiverged = (
+  local: Project,
+  remote: Project
+): string[] | null => {
+  let diverged: string[] | null = null;
+
+  const refs = local.cli.forked_from ?? {};
+
+  // for each workflow, check that the local fetched_from is the head of the remote history
+  for (const wf of local.workflows) {
+    if (wf.id in refs) {
+      const forkedVersion = refs[wf.id];
+      const remoteVersion = remote.getWorkflow(wf.id)?.history.at(-1);
+      if (forkedVersion !== remoteVersion) {
+        diverged ??= [];
+        diverged.push(wf.id);
+      }
+    } else {
+      // TODO what if there's no forked from for this workflow?
+      // Do we assume divergence because we don't know? Do we warn?
+    }
+  }
+
+  // TODO what if a workflow is removed locally?
+
+  return diverged;
+};
+
 export async function handler(options: DeployOptions, logger: Logger) {
   logger.warn(
     'WARNING: the project deploy command is in BETA and may not be stable. Use cautiously on production projects.'
@@ -122,6 +153,8 @@ Pass --force to override this error and deploy anyway.`);
     return false;
   }
 
+  // this fails now because the local project has no UUIDs
+  // But should that matter ,actually?
   const diffs = reportDiff(remoteProject!, localProject, logger);
   if (!diffs.length) {
     logger.success('Nothing to deploy');
@@ -132,32 +165,41 @@ Pass --force to override this error and deploy anyway.`);
 
   // Skip divergence testing if the remote has no history in its workflows
   // (this will only happen on older versions of lightning)
+  // TODO now maybe skip if there's no forked_from
   const skipVersionTest =
-    localProject.workflows.find((wf) => wf.history.length === 0) ||
+    // localProject.workflows.find((wf) => wf.history.length === 0) ||
     remoteProject.workflows.find((wf) => wf.history.length === 0);
 
+  // localProject.workflows.forEach((w) => console.log(w.history));
+
   if (skipVersionTest) {
     logger.warn(
       'Skipping compatibility check as no local version history detected'
     );
     logger.warn('Pushing these changes may overrite changes made to the app');
-  } else if (!localProject.canMergeInto(remoteProject!)) {
-    if (!options.force) {
-      logger.error(`Error: Projects have diverged!
+  } else {
+    const divergentWorkflows = hasRemoteDiverged(localProject, remoteProject!);
+    if (divergentWorkflows) {
+      logger.warn(
+        `The following workflows have diverged: ${divergentWorkflows}`
+      );
+      if (!options.force) {
+        logger.error(`Error: Projects have diverged!
 
-The remote project has been edited since the local project was branched. Changes may be lost.
+  The remote project has been edited since the local project was branched. Changes may be lost.
 
-Pass --force to override this error and deploy anyway.`);
-      return;
+  Pass --force to override this error and deploy anyway.`);
+        return;
+      } else {
+        logger.warn(
+          'Remote project has not diverged from local project! Pushing anyway as -f passed'
+        );
+      }
     } else {
-      logger.warn(
-        'Remote project has not diverged from local project! Pushing anyway as -f passed'
+      logger.info(
+        'Remote project has not diverged from local project - it is safe to deploy 🎉'
       );
     }
-  } else {
-    logger.info(
-      'Remote project has not diverged from local project - it is safe to deploy 🎉'
-    );
   }
 
   logger.info('Merging changes into remote project');
@@ -180,6 +222,8 @@ Pass --force to override this error and deploy anyway.`);
   // TODO not totally sold on endpoint handling right now
   config.endpoint ??= localProject.openfn?.endpoint!;
 
+  // TODO: I want to report diff HERE, after the merged state and stuff has been built
+
   if (options.dryRun) {
     logger.always('dryRun option set: skipping upload step');
   } else {
@@ -218,6 +262,14 @@ Pass --force to override this error and deploy anyway.`);
       merged.config
     );
 
+    // TODO why isn't this right? oh, because the outpu path isn't quite right
+    updateForkedFrom(finalProject);
+    const configData = finalProject.generateConfig();
+    await writeFile(
+      path.resolve(options.workspace!, configData.path),
+      configData.content
+    );
+
     const finalOutputPath = getSerializePath(localProject, options.workspace!);
     logger.debug('Updating local project at ', finalOutputPath);
     await serialize(finalProject, finalOutputPath);
@@ -267,3 +319,4 @@ export const reportDiff = (local: Project, remote: Project, logger: Logger) => {
 
   return diffs;
 };
+``;

packages/cli/src/projects/fetch.ts

@@ -331,9 +331,17 @@ To ignore this error and override the local file, pass --force (-f)
       options.force || // The user forced the checkout
       !hasAnyHistory; // the remote project has no history (can happen in old apps)
 
+    // TODO temporarily force skip
+    // TODO canMergeInto needs to return a reason
     if (!skipVersionCheck && !remoteProject.canMergeInto(localProject!)) {
       // TODO allow rename
-      throw new Error('Error! An incompatible project exists at this location');
+      const e = new Error(
+        `Error! An incompatible project exists at this location.`
+      );
+
+      delete e.stack;
+
+      throw e;
     }
   }
 }

packages/cli/src/projects/util.ts

@@ -216,3 +216,14 @@ export async function tidyWorkflowDir(
   // Return and sort for testing
   return toRemove.sort();
 }
+
+export const updateForkedFrom = (proj: Project) => {
+  proj.cli.forked_from = proj.workflows.reduce((obj: any, wf) => {
+    if (wf.history.length) {
+      obj[wf.id] = wf.history.at(-1);
+    }
+    return obj;
+  }, {});
+
+  return proj;
+};

packages/cli/test/projects/checkout.test.ts

@@ -3,7 +3,7 @@ import { createMockLogger } from '@openfn/logger';
 import { handler as checkoutHandler } from '../../src/projects/checkout';
 import mock from 'mock-fs';
 import fs from 'fs';
-import { jsonToYaml, Workspace } from '@openfn/project';
+import { jsonToYaml, Workspace, yamlToJson } from '@openfn/project';
 
 test.beforeEach(() => {
   mock({
@@ -28,6 +28,7 @@ test.beforeEach(() => {
         {
           name: 'simple-workflow',
           id: 'wf-id',
+          history: ['a'],
           jobs: [
             {
               name: 'Transform data to FHIR standard',
@@ -56,6 +57,7 @@ test.beforeEach(() => {
         {
           name: 'another-workflow',
           id: 'another-id',
+          history: ['b'],
           jobs: [
             {
               name: 'Transform data to FHIR standard',
@@ -83,13 +85,15 @@ test.beforeEach(() => {
         },
       ],
     }),
+    // TODO this is actually a v1 state file for some reason, which is wierd
     '/ws/.projects/project@app.openfn.org.yaml': jsonToYaml({
       id: '<uuid:main>',
       name: 'My Project',
       workflows: [
         {
           name: 'simple-workflow-main',
           id: 'wf-id-main',
+          version_history: ['a'],
           jobs: [
             {
               name: 'Transform data to FHIR standard',
@@ -118,6 +122,7 @@ test.beforeEach(() => {
         {
           name: 'another-workflow-main',
           id: 'another-id',
+          version_history: ['b'],
           jobs: [
             {
               name: 'Transform data to FHIR standard',
@@ -217,6 +222,25 @@ test.serial('checkout: same id as active', async (t) => {
   );
 });
 
+test.serial(
+  'checkout: writes forked_from based on version history',
+  async (t) => {
+    const bcheckout = new Workspace('/ws');
+    t.is(bcheckout.activeProject!.id, 'my-project');
+
+    await checkoutHandler(
+      { command: 'project-checkout', project: 'my-project', workspace: '/ws' },
+      logger
+    );
+
+    const openfn = yamlToJson(fs.readFileSync('/ws/openfn.yaml', 'utf8'));
+    t.deepEqual(openfn.project.forked_from, {
+      'simple-workflow-main': 'a',
+      'another-workflow-main': 'b',
+    });
+  }
+);
+
 test.serial('checkout: switching to and back between projects', async (t) => {
   // before checkout. my-project is active and expanded
   const bcheckout = new Workspace('/ws');

packages/cli/test/projects/deploy.test.ts

@@ -10,6 +10,7 @@ import createLightningServer, {
 
 import {
   handler as deployHandler,
+  hasRemoteDiverged,
   reportDiff,
 } from '../../src/projects/deploy';
 import { myProject_yaml, myProject_v1 } from './fixtures';
@@ -278,3 +279,53 @@ test.serial.skip(
     t.truthy(expectedLog);
   }
 );
+
+test('hasRemoteDiverged: 1 workflow, no diverged', (t) => {
+  const local = {
+    workflows: [
+      {
+        id: 'w',
+      },
+    ],
+    cli: {
+      forked_from: {
+        w: 'a',
+      },
+    },
+  } as unknown as Project;
+
+  const remote = {
+    getWorkflow: () => ({
+      id: 'w',
+      history: ['a'],
+    }),
+  } as unknown as Project;
+
+  const diverged = hasRemoteDiverged(local, remote);
+  t.falsy(diverged);
+});
+
+test('hasRemoteDiverged: 1 workflow, 1 diverged', (t) => {
+  const local = {
+    workflows: [
+      {
+        id: 'w',
+      },
+    ],
+    cli: {
+      forked_from: {
+        w: 'w',
+      },
+    },
+  } as unknown as Project;
+
+  const remote = {
+    getWorkflow: () => ({
+      id: 'w',
+      history: ['a', 'b'],
+    }),
+  } as unknown as Project;
+
+  const diverged = hasRemoteDiverged(local, remote);
+  t.deepEqual(diverged, ['w']);
+});

packages/cli/test/projects/fetch.test.ts

@@ -458,7 +458,7 @@ test.serial(
             lock_version: 1,
           },
           id: 'my-workflow',
-          history: ['cli:02582f3bb088'],
+          history: ['cli:ba19e179317f'],
         },
       ],
     };