diff --git a/forge/db/migrations/20260626-01-EE-extend-gittoken-generic.js b/forge/db/migrations/20260626-01-EE-extend-gittoken-generic.js new file mode 100644 index 0000000000..3f36aee4df --- /dev/null +++ b/forge/db/migrations/20260626-01-EE-extend-gittoken-generic.js @@ -0,0 +1,19 @@ +const { DataTypes } = require('sequelize') + +module.exports = { + /** + * upgrade database + * @param {QueryInterface} context Sequelize.QueryInterface + */ + up: async (context, Sequelize) => { + await context.addColumn('GitTokens', 'username', { + type: DataTypes.STRING, + allowNull: true + }) + await context.addColumn('GitTokens', 'caCertificate', { + type: DataTypes.TEXT, + allowNull: true + }) + }, + down: async (context, Sequelize) => { } +} diff --git a/forge/ee/db/models/GitToken.js b/forge/ee/db/models/GitToken.js index 32cc90fef0..67f1bc340c 100644 --- a/forge/ee/db/models/GitToken.js +++ b/forge/ee/db/models/GitToken.js @@ -15,6 +15,14 @@ module.exports = { type: DataTypes.STRING, allowNull: false, default: 'github' + }, + username: { + type: DataTypes.STRING, + allowNull: true + }, + caCertificate: { + type: DataTypes.TEXT, + allowNull: true } }, associations: function (M) { diff --git a/forge/ee/db/models/PipelineStageGitRepo.js b/forge/ee/db/models/PipelineStageGitRepo.js index 21453fc974..df40da2e1b 100644 --- a/forge/ee/db/models/PipelineStageGitRepo.js +++ b/forge/ee/db/models/PipelineStageGitRepo.js @@ -103,6 +103,8 @@ module.exports = { await app.gitops.pushToRepository({ token: gitToken.token, tokenType: gitToken.type, + username: gitToken.username, + caCertificate: gitToken.caCertificate, url: this.url, branch: this.branch, credentialSecret: this.credentialSecret, @@ -156,6 +158,8 @@ module.exports = { const snapshotContent = await app.gitops.pullFromRepository({ token: gitToken.token, tokenType: gitToken.type, + username: gitToken.username, + caCertificate: gitToken.caCertificate, url: this.url, branch: this.pullBranch || this.branch, credentialSecret: this.credentialSecret, diff --git a/forge/ee/db/views/PipelineStage.js b/forge/ee/db/views/PipelineStage.js index 4f746e80ec..7722e98784 100644 --- a/forge/ee/db/views/PipelineStage.js +++ b/forge/ee/db/views/PipelineStage.js @@ -28,10 +28,10 @@ module.exports = function (app) { pullBranch: { type: 'string' }, pushPath: { type: 'string' }, pullPath: { type: 'string' }, - lastPushAt: { type: 'string' }, - lastPullAt: { type: 'string' }, - status: { type: 'string' }, - statusMessage: { type: 'string' }, + lastPushAt: { type: 'string', nullable: true }, + lastPullAt: { type: 'string', nullable: true }, + status: { type: 'string', nullable: true }, + statusMessage: { type: 'string', nullable: true }, credentialSecret: { type: 'boolean' } }, required: ['gitTokenId', 'url', 'branch', 'pullBranch', 'pushPath', 'pullPath', 'lastPushAt', 'lastPullAt', 'status', 'statusMessage', 'credentialSecret'], diff --git a/forge/ee/lib/gitops/backends/generic.js b/forge/ee/lib/gitops/backends/generic.js new file mode 100644 index 0000000000..6b619e124c --- /dev/null +++ b/forge/ee/lib/gitops/backends/generic.js @@ -0,0 +1,195 @@ +const { execFile } = require('node:child_process') +const { existsSync } = require('node:fs') +const fs = require('node:fs/promises') +const os = require('node:os') +const path = require('node:path') +const { promisify } = require('node:util') +const execFilePromised = promisify(execFile) + +const { encryptValue, decryptValue } = require('../../../../db/utils') + +const { cloneRepository } = require('./utils') + +module.exports.init = async function (app) { + /** + * Push a snapshot to a git repository + * @param {Object} repoOptions + * @param {String} repoOptions.token + * @param {String} repoOptions.url + * @param {String} repoOptions.branch + * @param {String} repoOptions.username + * @param {Object} snapshot + * @param {Object} options + * @param {Object} options.sourceObject what produced the snapshot + * @param {Object} options.user who triggered the pipeline + * @param {Object} options.pipeline details of the pipeline + */ + async function pushToRepository (repoOptions, snapshot, options) { + let workingDir + let caFile + try { + const branch = repoOptions.branch || 'main' + if (!/^https:\/\//i.test(repoOptions.url)) { + throw new Error('Only HTTPS git URLs are supported') + } + const url = new URL(repoOptions.url) + url.username = repoOptions.username ? repoOptions.username : 'x-access-token' + url.password = repoOptions.token + + workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) + + let gitEnv = process.env + if (repoOptions.caCertificate) { + caFile = `${workingDir}-ca.pem` + await fs.writeFile(caFile, repoOptions.caCertificate) + gitEnv = { ...process.env, GIT_SSL_CAINFO: caFile } + } + + // 3. clone repo + await cloneRepository(url, branch, workingDir, gitEnv) + + // 4. set username/email + await execFilePromised('git', ['config', 'user.email', 'no-reply@flowfuse.com'], { cwd: workingDir, env: gitEnv }) + await execFilePromised('git', ['config', 'user.name', 'FlowFuse'], { cwd: workingDir, env: gitEnv }) + // For local dev - disable gpg signing in case its set in global config + await execFilePromised('git', ['config', 'commit.gpgsign', 'false'], { cwd: workingDir, env: gitEnv }) + + // 5. export snapshot + const exportOptions = { + credentialSecret: repoOptions.credentialSecret, + components: { + flows: true, + credentials: true + } + } + const result = await app.db.controllers.Snapshot.exportSnapshot(snapshot, exportOptions) + const snapshotExport = app.db.views.ProjectSnapshot.snapshotExport(result) + if (snapshotExport.settings?.settings?.palette?.npmrc) { + const enc = encryptValue(repoOptions.credentialSecret, snapshotExport.settings.settings?.palette?.npmrc) + snapshotExport.settings.settings.palette.npmrc = { $: enc } + } + const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') + await fs.writeFile(snapshotFile, JSON.stringify(snapshotExport, null, 4)) + + // 6. stage file + await execFilePromised('git', ['add', snapshotFile], { cwd: workingDir, env: gitEnv }) + + // 7. commit + const commitMessage = `Update snapshot\n\nSnapshot updated by FlowFuse Pipeline '${options.pipeline.name}', triggered by ${options.user.username}` + await execFilePromised('git', ['commit', '-m', commitMessage], { cwd: workingDir, env: gitEnv }) + + try { + // 8. push + await execFilePromised('git', ['push'], { cwd: workingDir, env: gitEnv }) + } catch (err) { + const output = err.stdout + err.stderr + if (/unable to access/.test(output)) { + const result = new Error('Permission denied') + result.code = 'invalid_token' + result.cause = err + throw result + } + let error + const m = /fatal: (.*)/.exec(output) + if (m) { + error = new Error('Failed to push repository: ' + m[1]) + } else { + error = Error('Failed to push repository') + } + error.cause = err + throw error + } + } finally { + if (workingDir) { + try { + await fs.rm(workingDir, { recursive: true, force: true }) + } catch (err) {} + } + if (caFile) { + try { + await fs.rm(caFile, { force: true }) + } catch (err) {} + } + } + } + + /** + * Push a snapshot to a git repository + * @param {Object} repoOptions + * @param {String} repoOptions.token + * @param {String} repoOptions.url + * @param {String} repoOptions.branch + * @param {String} repoOptions.username + */ + async function pullFromRepository (repoOptions) { + let workingDir + let caFile + try { + const branch = repoOptions.branch || 'main' + if (!/^https:\/\//i.test(repoOptions.url)) { + throw new Error('Only HTTPS git URLs are supported') + } + const url = new URL(repoOptions.url) + url.username = repoOptions.username ? repoOptions.username : 'x-access-token' + url.password = repoOptions.token + + workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-')) + + let gitEnv = process.env + if (repoOptions.caCertificate) { + caFile = `${workingDir}-ca.pem` + await fs.writeFile(caFile, repoOptions.caCertificate) + gitEnv = { ...process.env, GIT_SSL_CAINFO: caFile } + } + + // 3. clone repo + await cloneRepository(url, branch, workingDir, gitEnv) + + const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '') + + if (!existsSync(snapshotFile)) { + throw new Error('Snapshot file not found in repository') + } + + try { + const snapshotContent = await fs.readFile(snapshotFile, 'utf8') + const snapshot = JSON.parse(snapshotContent) + if (snapshot.settings?.env) { + const keys = Object.keys(snapshot.settings.env) + keys.forEach((key) => { + const env = snapshot.settings.env[key] + if (env.hidden && env.$) { + // Decrypt the value if it is encrypted + env.value = decryptValue(repoOptions.credentialSecret, env.$) + delete env.$ + } + }) + } + if (snapshot.settings?.settings?.palette?.npmrc) { + const npmrc = snapshot.settings.settings.palette.npmrc + if (typeof npmrc === 'object' && npmrc.$) { + snapshot.settings.settings.palette.npmrc = decryptValue(repoOptions.credentialSecret, npmrc.$) + } + } + return snapshot + } catch (err) { + throw new Error('Failed to read snapshot file: ' + err.message) + } + } finally { + if (workingDir) { + try { + await fs.rm(workingDir, { recursive: true, force: true }) + } catch (err) {} + } + if (caFile) { + try { + await fs.rm(caFile, { force: true }) + } catch (err) {} + } + } + } + return { + pushToRepository, + pullFromRepository + } +} diff --git a/forge/ee/lib/gitops/backends/utils.js b/forge/ee/lib/gitops/backends/utils.js index 7b069317f5..330251cbd9 100644 --- a/forge/ee/lib/gitops/backends/utils.js +++ b/forge/ee/lib/gitops/backends/utils.js @@ -1,10 +1,10 @@ -const { exec } = require('node:child_process') +const { execFile } = require('node:child_process') const { promisify } = require('node:util') -const execPromised = promisify(exec) +const execFilePromised = promisify(execFile) -async function cloneRepository (url, branch, workingDir) { +async function cloneRepository (url, branch, workingDir, env) { try { - await execPromised(`git clone -b ${branch} --depth 1 --single-branch ${url.toString()} .`, { cwd: workingDir }) + await execFilePromised('git', ['clone', '-b', branch, '--depth', '1', '--single-branch', url.toString(), '.'], { cwd: workingDir, env }) } catch (err) { const output = err.stdout + err.stderr // Token does not have access to clone the repo diff --git a/forge/ee/lib/gitops/index.js b/forge/ee/lib/gitops/index.js index 1850221528..a8789ec1a3 100644 --- a/forge/ee/lib/gitops/index.js +++ b/forge/ee/lib/gitops/index.js @@ -15,6 +15,7 @@ module.exports.init = async function (app) { // load backends const github = await require('./backends/github').init(app) const azure = await require('./backends/azure').init(app) + const generic = await require('./backends/generic').init(app) // Set the git feature flag app.config.features.register('gitIntegration', true, true) @@ -36,6 +37,8 @@ module.exports.init = async function (app) { await github.pushToRepository(repoOptions, snapshot, options) } else if (repoOptions.tokenType === 'azure') { await azure.pushToRepository(repoOptions, snapshot, options) + } else if (repoOptions.tokenType === 'generic') { + await generic.pushToRepository(repoOptions, snapshot, options) } } @@ -51,6 +54,8 @@ module.exports.init = async function (app) { return github.pullFromRepository(repoOptions) } else if (repoOptions.tokenType === 'azure') { return azure.pullFromRepository(repoOptions) + } else if (repoOptions.tokenType === 'generic') { + return generic.pullFromRepository(repoOptions) } } diff --git a/forge/ee/routes/gitops/index.js b/forge/ee/routes/gitops/index.js index 931047384f..9e1d2fd85d 100644 --- a/forge/ee/routes/gitops/index.js +++ b/forge/ee/routes/gitops/index.js @@ -49,7 +49,9 @@ module.exports = async function (app) { name: body.name, token: body.token, TeamId: request.team.id, - type: body.type || 'github' + type: body.type || 'github', + username: body.username || null, + caCertificate: body.caCertificate || null }) // TODO: audit log // await app.auditLog.Project.project.httpToken.created(request.session.User, null, request.project, body) diff --git a/frontend/src/components/icons/Git.js b/frontend/src/components/icons/Git.js index 393e110cfb..b93c5a9121 100644 --- a/frontend/src/components/icons/Git.js +++ b/frontend/src/components/icons/Git.js @@ -3,10 +3,9 @@ const { createVNode: _createVNode, openBlock: _openBlock, createBlock: _createBl module.exports = { props: { type: { - required: true, - type: String + type: String, + default: '' } - }, render: function (_ctx, _cache) { const path1 = _createVNode('path', { diff --git a/frontend/src/pages/application/PipelineStage/form.vue b/frontend/src/pages/application/PipelineStage/form.vue index 4d81102feb..eb36b087bf 100644 --- a/frontend/src/pages/application/PipelineStage/form.vue +++ b/frontend/src/pages/application/PipelineStage/form.vue @@ -144,7 +144,7 @@ Choose Git Token t.value === this.input.gitTokenId) + return tok?.type + }, gitPlaceholder () { - if (this.input.gitTokenId) { - for (const i in this.gitTokens) { - const tok = this.gitTokens[i] - if (tok.value === this.input.gitTokenId) { - if (tok.type === 'github') { - return 'e.g. https://github.com/[org]/[repo]' - } else if (tok.type === 'azure') { - return 'e.g. https://dev.azure.com/[org]/_git/[repo]' - } - } - } + if (this.selectedGitTokenType === 'azure') { + return 'e.g. https://dev.azure.com/[org]/_git/[repo]' + } else if (this.selectedGitTokenType === 'generic') { + return 'e.g. https://git.example.com/org/repo.git' } return 'e.g. https://github.com/[org]/[repo]' } @@ -685,14 +682,11 @@ export default { // If not, reset to the stages original action (if available) this.input.action = this.stage?.action && this.actionOptions.some((option) => option.value === this.stage.action) ? this.stage.action : null }, - 'input.url' (newUrl, oldUrl) { - if (newUrl === '') { - this.errors.url = '' - } else if (!/^https:\/\/github\.com\/[^/]+\/[^/]+$/.test(newUrl) && !/^https:\/\/dev\.azure\.com\/[^/]+\/_git\/[^/]+$/.test(newUrl)) { - this.errors.url = 'Please enter a valid GitHub or Azure DevOps repository URL' - } else { - this.errors.url = '' - } + 'input.url' () { + this.validateGitUrl() + }, + 'input.gitTokenId' () { + this.validateGitUrl() }, 'input.pushPath' (newPushPath, oldPushPath) { if (newPushPath === '' && this.isFirstStage) { @@ -726,6 +720,19 @@ export default { this.original.deviceGroupId = this.input.deviceGroupId }, methods: { + validateGitUrl () { + const url = this.input.url + const type = this.selectedGitTokenType + if (url === '') { + this.errors.url = '' + } else if (type === 'github' || type === 'azure') { + this.errors.url = (/^https:\/\/github\.com\/[^/]+\/[^/]+$/.test(url) || /^https:\/\/dev\.azure\.com\/[^/]+\/_git\/[^/]+$/.test(url)) + ? '' + : 'Please enter a valid GitHub or Azure DevOps repository URL' + } else { + this.errors.url = /^https:\/\//i.test(url) ? '' : 'Please enter a valid HTTPS repository URL' + } + }, async submit () { this.loading.creating = !this.isEdit this.loading.updating = this.isEdit diff --git a/frontend/src/pages/team/Settings/Integrations.vue b/frontend/src/pages/team/Settings/Integrations.vue index 266ce097e4..30ebee4bd6 100644 --- a/frontend/src/pages/team/Settings/Integrations.vue +++ b/frontend/src/pages/team/Settings/Integrations.vue @@ -1,7 +1,7 @@