[5294] Git Integration in a DevOps Pipeline

forge/db/migrations/20260626-01-EE-extend-gittoken-generic.js

@@ -0,0 +1,19 @@
+const { DataTypes } = require('sequelize')
+
+module.exports = {
+    /**
+     * upgrade database
+     * @param {QueryInterface} context Sequelize.QueryInterface
+     */
+    up: async (context, Sequelize) => {
+        await context.addColumn('GitTokens', 'username', {
+            type: DataTypes.STRING,
+            allowNull: true
+        })
+        await context.addColumn('GitTokens', 'caCertificate', {
+            type: DataTypes.TEXT,
+            allowNull: true
+        })
+    },
+    down: async (context, Sequelize) => { }
+}

forge/ee/db/models/GitToken.js

@@ -15,6 +15,14 @@ module.exports = {
             type: DataTypes.STRING,
             allowNull: false,
             default: 'github'
+        },
+        username: {
+            type: DataTypes.STRING,
+            allowNull: true
+        },
+        caCertificate: {
+            type: DataTypes.TEXT,
+            allowNull: true
         }
     },
     associations: function (M) {

forge/ee/db/models/PipelineStageGitRepo.js

@@ -103,6 +103,8 @@ module.exports = {
                             await app.gitops.pushToRepository({
                                 token: gitToken.token,
                                 tokenType: gitToken.type,
+                                username: gitToken.username,
+                                caCertificate: gitToken.caCertificate,
                                 url: this.url,
                                 branch: this.branch,
                                 credentialSecret: this.credentialSecret,
@@ -156,6 +158,8 @@ module.exports = {
                         const snapshotContent = await app.gitops.pullFromRepository({
                             token: gitToken.token,
                             tokenType: gitToken.type,
+                            username: gitToken.username,
+                            caCertificate: gitToken.caCertificate,
                             url: this.url,
                             branch: this.pullBranch || this.branch,
                             credentialSecret: this.credentialSecret,

forge/ee/db/views/PipelineStage.js

@@ -28,10 +28,10 @@ module.exports = function (app) {
                     pullBranch: { type: 'string' },
                     pushPath: { type: 'string' },
                     pullPath: { type: 'string' },
-                    lastPushAt: { type: 'string' },
-                    lastPullAt: { type: 'string' },
-                    status: { type: 'string' },
-                    statusMessage: { type: 'string' },
+                    lastPushAt: { type: 'string', nullable: true },
+                    lastPullAt: { type: 'string', nullable: true },
+                    status: { type: 'string', nullable: true },
+                    statusMessage: { type: 'string', nullable: true },
                     credentialSecret: { type: 'boolean' }
                 },
                 required: ['gitTokenId', 'url', 'branch', 'pullBranch', 'pushPath', 'pullPath', 'lastPushAt', 'lastPullAt', 'status', 'statusMessage', 'credentialSecret'],

forge/ee/lib/gitops/backends/generic.js

@@ -0,0 +1,195 @@
+const { execFile } = require('node:child_process')
+const { existsSync } = require('node:fs')
+const fs = require('node:fs/promises')
+const os = require('node:os')
+const path = require('node:path')
+const { promisify } = require('node:util')
+const execFilePromised = promisify(execFile)
+
+const { encryptValue, decryptValue } = require('../../../../db/utils')
+
+const { cloneRepository } = require('./utils')
+
+module.exports.init = async function (app) {
+    /**
+     * Push a snapshot to a git repository
+     * @param {Object} repoOptions
+     * @param {String} repoOptions.token
+     * @param {String} repoOptions.url
+     * @param {String} repoOptions.branch
+     * @param {String} repoOptions.username
+     * @param {Object} snapshot
+     * @param {Object} options
+     * @param {Object} options.sourceObject what produced the snapshot
+     * @param {Object} options.user who triggered the pipeline
+     * @param {Object} options.pipeline details of the pipeline
+     */
+    async function pushToRepository (repoOptions, snapshot, options) {
+        let workingDir
+        let caFile
+        try {
+            const branch = repoOptions.branch || 'main'
+            if (!/^https:\/\//i.test(repoOptions.url)) {
+                throw new Error('Only HTTPS git URLs are supported')
+            }
+            const url = new URL(repoOptions.url)
+            url.username = repoOptions.username ? repoOptions.username : 'x-access-token'
+            url.password = repoOptions.token
+
+            workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-'))
+
+            let gitEnv = process.env
+            if (repoOptions.caCertificate) {
+                caFile = `${workingDir}-ca.pem`
+                await fs.writeFile(caFile, repoOptions.caCertificate)
+                gitEnv = { ...process.env, GIT_SSL_CAINFO: caFile }
+            }
+
+            // 3. clone repo
+            await cloneRepository(url, branch, workingDir, gitEnv)
+
+            // 4. set username/email
+            await execFilePromised('git', ['config', 'user.email', 'no-reply@flowfuse.com'], { cwd: workingDir, env: gitEnv })
+            await execFilePromised('git', ['config', 'user.name', 'FlowFuse'], { cwd: workingDir, env: gitEnv })
+            // For local dev - disable gpg signing in case its set in global config
+            await execFilePromised('git', ['config', 'commit.gpgsign', 'false'], { cwd: workingDir, env: gitEnv })
+
+            // 5. export snapshot
+            const exportOptions = {
+                credentialSecret: repoOptions.credentialSecret,
+                components: {
+                    flows: true,
+                    credentials: true
+                }
+            }
+            const result = await app.db.controllers.Snapshot.exportSnapshot(snapshot, exportOptions)
+            const snapshotExport = app.db.views.ProjectSnapshot.snapshotExport(result)
+            if (snapshotExport.settings?.settings?.palette?.npmrc) {
+                const enc = encryptValue(repoOptions.credentialSecret, snapshotExport.settings.settings?.palette?.npmrc)
+                snapshotExport.settings.settings.palette.npmrc = { $: enc }
+            }
+            const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '')
+            await fs.writeFile(snapshotFile, JSON.stringify(snapshotExport, null, 4))
+
+            // 6. stage file
+            await execFilePromised('git', ['add', snapshotFile], { cwd: workingDir, env: gitEnv })
+
+            // 7. commit
+            const commitMessage = `Update snapshot\n\nSnapshot updated by FlowFuse Pipeline '${options.pipeline.name}', triggered by ${options.user.username}`
+            await execFilePromised('git', ['commit', '-m', commitMessage], { cwd: workingDir, env: gitEnv })
+
+            try {
+                // 8. push
+                await execFilePromised('git', ['push'], { cwd: workingDir, env: gitEnv })
+            } catch (err) {
+                const output = err.stdout + err.stderr
+                if (/unable to access/.test(output)) {
+                    const result = new Error('Permission denied')
+                    result.code = 'invalid_token'
+                    result.cause = err
+                    throw result
+                }
+                let error
+                const m = /fatal: (.*)/.exec(output)
+                if (m) {
+                    error = new Error('Failed to push repository: ' + m[1])
+                } else {
+                    error = Error('Failed to push repository')
+                }
+                error.cause = err
+                throw error
+            }
+        } finally {
+            if (workingDir) {
+                try {
+                    await fs.rm(workingDir, { recursive: true, force: true })
+                } catch (err) {}
+            }
+            if (caFile) {
+                try {
+                    await fs.rm(caFile, { force: true })
+                } catch (err) {}
+            }
+        }
+    }
+
+    /**
+     * Push a snapshot to a git repository
+     * @param {Object} repoOptions
+     * @param {String} repoOptions.token
+     * @param {String} repoOptions.url
+     * @param {String} repoOptions.branch
+     * @param {String} repoOptions.username
+     */
+    async function pullFromRepository (repoOptions) {
+        let workingDir
+        let caFile
+        try {
+            const branch = repoOptions.branch || 'main'
+            if (!/^https:\/\//i.test(repoOptions.url)) {
+                throw new Error('Only HTTPS git URLs are supported')
+            }
+            const url = new URL(repoOptions.url)
+            url.username = repoOptions.username ? repoOptions.username : 'x-access-token'
+            url.password = repoOptions.token
+
+            workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flowfuse-git-repo-'))
+
+            let gitEnv = process.env
+            if (repoOptions.caCertificate) {
+                caFile = `${workingDir}-ca.pem`
+                await fs.writeFile(caFile, repoOptions.caCertificate)
+                gitEnv = { ...process.env, GIT_SSL_CAINFO: caFile }
+            }
+
+            // 3. clone repo
+            await cloneRepository(url, branch, workingDir, gitEnv)
+
+            const snapshotFile = path.join(workingDir, repoOptions.path || 'snapshot.json').replace(/"/g, '')
+
+            if (!existsSync(snapshotFile)) {
+                throw new Error('Snapshot file not found in repository')
+            }
+
+            try {
+                const snapshotContent = await fs.readFile(snapshotFile, 'utf8')
+                const snapshot = JSON.parse(snapshotContent)
+                if (snapshot.settings?.env) {
+                    const keys = Object.keys(snapshot.settings.env)
+                    keys.forEach((key) => {
+                        const env = snapshot.settings.env[key]
+                        if (env.hidden && env.$) {
+                            // Decrypt the value if it is encrypted
+                            env.value = decryptValue(repoOptions.credentialSecret, env.$)
+                            delete env.$
+                        }
+                    })
+                }
+                if (snapshot.settings?.settings?.palette?.npmrc) {
+                    const npmrc = snapshot.settings.settings.palette.npmrc
+                    if (typeof npmrc === 'object' && npmrc.$) {
+                        snapshot.settings.settings.palette.npmrc = decryptValue(repoOptions.credentialSecret, npmrc.$)
+                    }
+                }
+                return snapshot
+            } catch (err) {
+                throw new Error('Failed to read snapshot file: ' + err.message)
+            }
+        } finally {
+            if (workingDir) {
+                try {
+                    await fs.rm(workingDir, { recursive: true, force: true })
+                } catch (err) {}
+            }
+            if (caFile) {
+                try {
+                    await fs.rm(caFile, { force: true })
+                } catch (err) {}
+            }
+        }
+    }
+    return {
+        pushToRepository,
+        pullFromRepository
+    }
+}

forge/ee/lib/gitops/backends/utils.js

@@ -1,10 +1,10 @@
-const { exec } = require('node:child_process')
+const { execFile } = require('node:child_process')
 const { promisify } = require('node:util')
-const execPromised = promisify(exec)
+const execFilePromised = promisify(execFile)
 
-async function cloneRepository (url, branch, workingDir) {
+async function cloneRepository (url, branch, workingDir, env) {
     try {
-        await execPromised(`git clone -b ${branch} --depth 1 --single-branch ${url.toString()} .`, { cwd: workingDir })
+        await execFilePromised('git', ['clone', '-b', branch, '--depth', '1', '--single-branch', url.toString(), '.'], { cwd: workingDir, env })
     } catch (err) {
         const output = err.stdout + err.stderr
         // Token does not have access to clone the repo

forge/ee/lib/gitops/index.js

@@ -15,6 +15,7 @@ module.exports.init = async function (app) {
     // load backends
     const github = await require('./backends/github').init(app)
     const azure = await require('./backends/azure').init(app)
+    const generic = await require('./backends/generic').init(app)
 
     // Set the git feature flag
     app.config.features.register('gitIntegration', true, true)
@@ -36,6 +37,8 @@ module.exports.init = async function (app) {
             await github.pushToRepository(repoOptions, snapshot, options)
         } else if (repoOptions.tokenType === 'azure') {
             await azure.pushToRepository(repoOptions, snapshot, options)
+        } else if (repoOptions.tokenType === 'generic') {
+            await generic.pushToRepository(repoOptions, snapshot, options)
         }
     }
 
@@ -51,6 +54,8 @@ module.exports.init = async function (app) {
             return github.pullFromRepository(repoOptions)
         } else if (repoOptions.tokenType === 'azure') {
             return azure.pullFromRepository(repoOptions)
+        } else if (repoOptions.tokenType === 'generic') {
+            return generic.pullFromRepository(repoOptions)
         }
     }
 

forge/ee/routes/gitops/index.js

@@ -49,7 +49,9 @@ module.exports = async function (app) {
                 name: body.name,
                 token: body.token,
                 TeamId: request.team.id,
-                type: body.type || 'github'
+                type: body.type || 'github',
+                username: body.username || null,
+                caCertificate: body.caCertificate || null
             })
             // TODO: audit log
             // await app.auditLog.Project.project.httpToken.created(request.session.User, null, request.project, body)

frontend/src/components/icons/Git.js

@@ -3,10 +3,9 @@ const { createVNode: _createVNode, openBlock: _openBlock, createBlock: _createBl
 module.exports = {
     props: {
         type: {
-            required: true,
-            type: String
+            type: String,
+            default: ''
         }
-
     },
     render: function (_ctx, _cache) {
         const path1 = _createVNode('path', {