Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .tool-versions
Original file line number Diff line number Diff line change
@@ -1 +1 @@
nodejs 24.14.0
nodejs 24.15.0
8 changes: 8 additions & 0 deletions integration-tests/cli/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
# @openfn/integration-tests-cli

## 1.0.22

### Patch Changes

- Updated dependencies
- @openfn/project@0.15.2
- @openfn/lightning-mock@2.4.17

## 1.0.21

### Patch Changes
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/cli/package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@openfn/integration-tests-cli",
"private": true,
"version": "1.0.21",
"version": "1.0.22",
"description": "CLI integration tests",
"author": "Open Function Group <admin@openfn.org>",
"license": "ISC",
Expand Down
3 changes: 1 addition & 2 deletions integration-tests/cli/test/project-v1.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,6 @@ test.serial('Checkout a project', async (t) => {
`id: my-workflow
name: my workflow
start: webhook
options: {}
steps:
- id: webhook
type: webhook
Expand All @@ -153,7 +152,7 @@ steps:
condition: always
- id: transform-data
name: Transform data
adaptor: "@openfn/language-common@latest"
adaptor: '@openfn/language-common@latest'
expression: ./transform-data.js
`
);
Expand Down
3 changes: 1 addition & 2 deletions integration-tests/cli/test/project-v2.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,6 @@ test.serial('Checkout a project', async (t) => {
`id: hello-workflow
name: Hello Workflow
start: trigger
options: {}
steps:
- id: trigger
type: webhook
Expand All @@ -161,7 +160,7 @@ steps:
condition: true
- id: transform-data
name: Transform data
adaptor: "@openfn/language-dhis2@8.0.4"
adaptor: '@openfn/language-dhis2@8.0.4'
expression: ./transform-data.js
`
);
Expand Down
10 changes: 10 additions & 0 deletions packages/cli/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
# @openfn/cli

## 1.35.3

### Patch Changes

- Include version number in serialized project files
- Don't validate workflows on execution unless `--validate` is explicitly passed
- Allow setting `PREFER_LEGACY_SYNC=1` to strictly disable v2 sync when calling the v1 API
- Updated dependencies
- @openfn/project@0.15.2

## 1.35.2

### Patch Changes
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@openfn/cli",
"version": "1.35.2",
"version": "1.35.3",
"description": "CLI devtools for the OpenFn toolchain",
"engines": {
"node": ">=18",
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/deploy/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ async function deployHandler(
options.workspace || process.cwd(),
'openfn.yaml'
);
if (await fileExists(v2ConfigPath)) {
if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) {
// override endpoint with one from openfn.yaml
const config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8'));
if (config?.project?.endpoint) {
config.endpoint = config.project.endpoint;
}

logger.always(
'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy)'
'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.'
);
return beta.handler(
{
Expand Down
22 changes: 12 additions & 10 deletions packages/cli/src/execute/command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,39 +8,40 @@ import type { Opts } from '../options';
export type ExecuteOptions = { workspace?: string } & Required<
Pick<
Opts,
| 'apiKey'
| 'adaptors'
| 'apiKey'
| 'autoinstall'
| 'baseDir'
| 'cacheSteps'
| 'collectionsEndpoint'
| 'collectionsVersion'
| 'command'
| 'compile'
| 'credentials'
| 'collectionsEndpoint'
| 'collectionsVersion'
| 'expandAdaptors'
| 'end'
| 'immutable'
| 'ignoreImports'
| 'expandAdaptors'
| 'expressionPath'
| 'globals'
| 'ignoreImports'
| 'immutable'
| 'log'
| 'logJson'
| 'only'
| 'outputPath'
| 'outputStdout'
| 'only'
| 'path'
| 'repoDir'
| 'sanitize'
| 'skipAdaptorValidation'
| 'start'
| 'statePath'
| 'stateStdin'
| 'sanitize'
| 'timeout'
| 'trace'
| 'useAdaptorsMonorepo'
| 'workflowPath'
| 'validate'
| 'workflowName'
| 'globals'
| 'workflowPath'
>
> &
Pick<Opts, 'monorepoPath' | 'repoDir'>;
Expand Down Expand Up @@ -77,6 +78,7 @@ const options = [
o.timeout,
o.trace,
o.useAdaptorsMonorepo,
o.validate,

po.workspace,
];
Expand Down
4 changes: 3 additions & 1 deletion packages/cli/src/execute/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@ const executeHandler = async (options: ExecuteOptions, logger: Logger) => {
await validateAdaptors(options, logger);

let plan = await loadPlan(options, logger);
validatePlan(plan, logger);
if (options.validate) {
validatePlan(plan, logger);
}
await loadAndApplyCredentialMap(plan, options, logger);
if (options.cacheSteps) {
await clearCache(plan, options, logger);
Expand Down
10 changes: 10 additions & 0 deletions packages/cli/src/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export type Opts = {
useAdaptorsMonorepo?: boolean;
workflow: string;
workflowName?: string;
validate?: boolean;

// deprecated
workflowPath?: string;
Expand Down Expand Up @@ -613,6 +614,15 @@ export const sanitize: CLIOption = {
},
};

export const validate: CLIOption = {
name: 'validate',
yargs: {
boolean: true,
default: false,
description: 'Validate workflows before executing',
},
};

export const workflow: CLIOption = {
name: 'workflow',
yargs: {
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/pull/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ async function pullHandler(options: PullOptions, logger: Logger) {
options.workspace || process.cwd(),
'openfn.yaml'
);
if (await fileExists(v2ConfigPath)) {
if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) {
// override endpoint with one from openfn.yaml
const config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8'));
if (config?.project?.endpoint) {
config.endpoint = config.project.endpoint;
}

logger.always(
'Detected openfn.yaml file - switching to v2 pull (openfn project pull)'
'Detected openfn.yaml file - switching to v2 pull (openfn project pull). Set PREFER_LEGACY_SYNC to disable this.'
);
return beta(
{
Expand Down
1 change: 0 additions & 1 deletion packages/cli/test/projects/checkout.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,6 @@ workspace:
t.deepEqual(JSON.parse(wf), {
id: 'simple-workflow',
name: 'Simple Workflow',
options: {},
start: 'webhook',
steps: [
{
Expand Down
4 changes: 1 addition & 3 deletions packages/cli/test/projects/fetch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -402,9 +402,7 @@ test.serial(
const json = {
id: 'my-project',
name: 'My Project',
cli: {
version: 2,
},
schema_version: '4.0',
description: 'my lovely project',
collections: [],
credentials: [],
Expand Down
5 changes: 2 additions & 3 deletions packages/cli/test/projects/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,7 @@ export const myProject_v1: Provisioner.Project = {

export const myProject_yaml = `id: my-project
name: My Project
cli:
version: 2
schema_version: '4.0'
description: my lovely project
collections: []
credentials: []
Expand All @@ -81,7 +80,7 @@ workflows:
- id: transform-data
name: Transform data
expression: fn()
adaptor: "@openfn/language-common@latest"
adaptor: '@openfn/language-common@latest'
openfn:
uuid: 66add020-e6eb-4eec-836b-20008afca816
- id: webhook
Expand Down
12 changes: 7 additions & 5 deletions packages/lexicon/portability.d.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
/**
* Typings which define the Portability Spec
* See https://docs.openfn.org/documentation/deploy/portability
*/

/**
* Schema for portable representation of a Project
* AKA Project Spec
* Can serialize to JSON or YAML or whatever you like
*
* This interface describes the 4.0 Portability Spec: https://docs.openfn.org/documentation/deploy/portability
*
* If you serialize a v2 project file without state, this is what you get
*/

export interface ProjectSpec {
/** Single-word identifier */
id: string;
Expand All @@ -19,6 +17,8 @@ export interface ProjectSpec {

description?: string;

schema_version?: string;

workflows: WorkflowSpec[];

credentials?: Credential[];
Expand All @@ -33,6 +33,8 @@ export interface WorkflowSpec {
/** Human readable label. Can be used to generate an internal id */
name?: string;

schema_version?: string;

steps: Array<Job | Trigger>;

/** Global functions (path to a js file, unsupported in app) */
Expand Down
7 changes: 7 additions & 0 deletions packages/lightning-mock/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# @openfn/lightning-mock

## 2.4.17

### Patch Changes

- Updated dependencies
- @openfn/project@0.15.2

## 2.4.16

### Patch Changes
Expand Down
2 changes: 1 addition & 1 deletion packages/lightning-mock/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@openfn/lightning-mock",
"version": "2.4.16",
"version": "2.4.17",
"private": true,
"description": "A mock Lightning server",
"main": "dist/index.js",
Expand Down
6 changes: 6 additions & 0 deletions packages/project/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# @openfn/project

## 0.15.2

### Patch Changes

- Include version number in serialized project files

## 0.15.1

### Patch Changes
Expand Down
2 changes: 1 addition & 1 deletion packages/project/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@openfn/project",
"version": "0.15.1",
"version": "0.15.2",
"description": "Read, serialize, replicate and sync OpenFn projects",
"scripts": {
"test": "pnpm ava",
Expand Down
22 changes: 7 additions & 15 deletions packages/project/src/parse/from-project.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,15 @@ import Project from '../Project';
import ensureJson from '../util/ensure-json';
import { Provisioner } from '@openfn/lexicon/lightning';
import fromAppState, { fromAppStateConfig } from './from-app-state';
import { WithMeta } from '../Workflow';

// Load a project from any JSON or yaml representation
// This is backwards-compatible with v1 state.json files
// But is really designed for v2 project.yaml files

// TODO move these types to a common types.ts, or maybe Project.ts
export type SerializedProject = Omit<Partial<l.ProjectState>, 'workflows'> & {
version: number;
workflows: SerializedWorkflow[];
};

export type SerializedWorkflow = {
id: string;
name: string;
export type SerializedProject = l.ProjectState;

steps: WithMeta<Array<l.Job | l.Trigger>>;

openfn?: l.ProjectMeta;
};
export type SerializedWorkflow = l.WorkflowState;

export default (
data: l.ProjectState | SerializedProject | string,
Expand All @@ -32,8 +21,11 @@ export default (
// first ensure the data is in JSON format
let rawJson = ensureJson<any>(data);

if (rawJson.cli?.version ?? rawJson.version /*deprecated*/) {
// If there's any version key at all, its at least v2
if (
rawJson.schema_version ||
rawJson.cli?.version === 2 ||
rawJson.version /*deprecated*/
) {
return new Project(from_v2(rawJson as SerializedProject), config);
}

Expand Down
Loading