From 5fac24ea9daa710e6c07df3c7a4ef4d682e0e6e2 Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Tue, 30 Aug 2022 10:54:59 -0400 Subject: [PATCH] Add rollback This commit adds simple rollback by trying to undo the changes made during a failed Apply by Applying in the opposite direction. Signed-off-by: Joseph Sawaya --- .github/workflows/docker-image.yml | 100 +++++++++++++++++++++++++++++ examples/ci-rollback.yaml | 10 +++ examples/raw/cap.yaml | 25 ++++---- examples/rollback/1-working.yaml | 9 +++ examples/rollback/2-broken.yaml | 10 +++ examples/rollback/working.yaml | 9 +++ go.mod | 2 +- pkg/engine/apply.go | 25 ++++++++ pkg/engine/common.go | 45 ++++++++++++- pkg/engine/types.go | 4 ++ 10 files changed, 222 insertions(+), 17 deletions(-) create mode 100644 examples/ci-rollback.yaml create mode 100644 examples/rollback/1-working.yaml create mode 100644 examples/rollback/2-broken.yaml create mode 100644 examples/rollback/working.yaml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 07c30fc7..80a246e9 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1130,6 +1130,106 @@ jobs: - name: Check the capabilities of cap1 run: if [[ $(sudo podman container inspect cap1 --format {{.EffectiveCaps}} | grep NET_ADMIN | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi + rollback-validate: + runs-on: ubuntu-latest + if: > + (github.event_name == 'push' || github.event_name == 'schedule') && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + needs: [ build , pull-and-archive ] + steps: + - name: pull in podman + uses: actions/download-artifact@v1 + with: + name: podman-bins + path: bin + + - name: replace + run: | + chmod +x bin/podman + sudo mv bin/podman /usr/bin/podman + + - name: Enable the podman socket + run: sudo systemctl enable --now podman.socket + + - uses: actions/checkout@v3 + with: + path: main + + - name: checkout with token + uses: actions/checkout@v3 + with: + path: ci-rollback + ref: ci-rollback + + - name: Enable the podman socket + run: sudo systemctl enable --now podman.socket + + - name: reset and commit working + run: | + cd ci-rollback + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + rm -rf ./examples/ci-rollback || true + mkdir ./examples/ci-rollback + cp ./examples/rollback/working.yaml ./examples/ci-rollback/working.yaml + git add ./examples + git commit -m "add working" -a || true + git push -f || true + + - name: pull artifact + uses: actions/download-artifact@v1 + with: + name: fetchit-image + path: /tmp + + - name: Load the image + run: sudo podman load -i /tmp/fetchit.tar + + - name: pull artifact + uses: actions/download-artifact@v1 + with: + name: colors + path: /tmp + + - name: Load the image + run: sudo podman load -i /tmp/colors.tar + + - name: tag the image + run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest + + - name: set values relating to the current env + run: | + cd ci-rollback + sed -i 's| url: http://github.com/containers/fetchit| url: http://github.com/${{ github.repository }}|g' ./examples/ci-rollback.yaml + sed -i 's| branch: ci-rollback| branch: "{{ github.ref }}"|g' ./examples/ci-rollback.yaml + - name: Start fetchit + run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /home/runner/work/fetchit/fetchit/ci-rollback/examples/ci-rollback.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest + + - name: identify rollback1 container + run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do sudo podman ps; c=$(sudo podman ps | grep rollback | wc -l); done' + + - name: Logs + if: always() + run: sudo podman logs fetchit + + - name: commit working + run: | + cd ci-rollback + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + ls + cp ./examples/rollback/1-working.yaml ./examples/ci-rollback/1-working.yaml + cp ./examples/rollback/2-broken.yaml ./examples/ci-rollback/2-broken.yaml + git add ./examples + git commit -m "add 1-working and 2-broken" -a + git push -f + + - name: Wait for 2 pods + run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do sudo podman ps; c=$(sudo podman ps | grep rollback | wc -l); done' + + - name: Check rollback succeeded + run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman ps | grep rollback | wc -l); done' + multi-engine-validate: runs-on: ubuntu-latest needs: [ build , pull-and-archive ] diff --git a/examples/ci-rollback.yaml b/examples/ci-rollback.yaml new file mode 100644 index 00000000..fcd9438a --- /dev/null +++ b/examples/ci-rollback.yaml @@ -0,0 +1,10 @@ +targetConfigs: +- name: fetchit + url: http://github.com/containers/fetchit + raw: + - name: raw + targetPath: examples/ci-rollback + schedule: "*/1 * * * *" + rollback: true + trackBadCommits: true + branch: ci-rollback \ No newline at end of file diff --git a/examples/raw/cap.yaml b/examples/raw/cap.yaml index 690417cc..2543ebc1 100644 --- a/examples/raw/cap.yaml +++ b/examples/raw/cap.yaml @@ -1,14 +1,11 @@ -{ - "Image":"docker.io/mmumshad/simple-webapp-color:latest", - "Name": "cap2", - "Env": {"APP_COLOR": "blue", "tree": "trunk"}, - "Mounts": [], - "Volumes": [], - "CapDrop": ["all"], - "Ports": [{ - "host_ip": "", - "container_port": 8080, - "host_port": 9090, - "range": 0, - "protocol": ""}] -} +Image: docker.io/mmumshad/simple-webapp-color:latest +Name: cap2 +Env: + APP_COLOR": blue + tree: trunk +CapDrop: + - all +Ports: + - container_port: 8080 + host_port: 9091 + range: 0 \ No newline at end of file diff --git a/examples/rollback/1-working.yaml b/examples/rollback/1-working.yaml new file mode 100644 index 00000000..d4959df0 --- /dev/null +++ b/examples/rollback/1-working.yaml @@ -0,0 +1,9 @@ +Image: "docker.io/mmumshad/simple-webapp-color:latest" +Name: "rollback2" +Env: + APP_COLOR: "pink" + tree: "trunk" +Ports: + - container_port: 8080 + host_port: 8081 + range: 0 \ No newline at end of file diff --git a/examples/rollback/2-broken.yaml b/examples/rollback/2-broken.yaml new file mode 100644 index 00000000..90d54544 --- /dev/null +++ b/examples/rollback/2-broken.yaml @@ -0,0 +1,10 @@ +# Typo below +Image: "docker.io/mmumshad/simple-webapp-clor:latest" +Name: "rollback3" +Env: + APP_COLOR: "pink" + tree: "trunk" +Ports: + - container_port: 8080 + host_port: 8080 + range: 0 \ No newline at end of file diff --git a/examples/rollback/working.yaml b/examples/rollback/working.yaml new file mode 100644 index 00000000..f63d101d --- /dev/null +++ b/examples/rollback/working.yaml @@ -0,0 +1,9 @@ +Image: "docker.io/mmumshad/simple-webapp-color:latest" +Name: "rollback1" +Env: + APP_COLOR: "pink" + tree: "trunk" +Ports: + - container_port: 8080 + host_port: 8080 + range: 0 \ No newline at end of file diff --git a/go.mod b/go.mod index 77c81ddd..76d2235f 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.23.5 k8s.io/apimachinery v0.23.5 + k8s.io/klog/v2 v2.60.1-0.20220317184644-43cc75f9ae89 sigs.k8s.io/yaml v1.3.0 ) @@ -250,7 +251,6 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/klog/v2 v2.60.1-0.20220317184644-43cc75f9ae89 // indirect k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect diff --git a/pkg/engine/apply.go b/pkg/engine/apply.go index 35443c96..d5a70eb1 100644 --- a/pkg/engine/apply.go +++ b/pkg/engine/apply.go @@ -153,6 +153,31 @@ func certHexFingerprint(cert *x509.Certificate) string { return hex.EncodeToString(fpr[:]) } +func checkout(target *Target, hash plumbing.Hash) error { + if hash == plumbing.ZeroHash { + return nil + } + + directory := getDirectory(target) + + repo, err := git.PlainOpen(directory) + if err != nil { + return utils.WrapErr(err, "Error opening repository: %s", directory) + } + + wt, err := repo.Worktree() + if err != nil { + return utils.WrapErr(err, "Error getting reference to worktree for repository", directory) + } + + err = wt.Checkout(&git.CheckoutOptions{Hash: hash}) + if err != nil { + return utils.WrapErr(err, "Error checking out %s on branch %s", hash, target.branch) + } + + return nil +} + func getCurrent(target *Target, methodType, methodName string) (plumbing.Hash, error) { directory := getDirectory(target) tagName := fmt.Sprintf("current-%s-%s", methodType, methodName) diff --git a/pkg/engine/common.go b/pkg/engine/common.go index 78430d4b..434ae932 100644 --- a/pkg/engine/common.go +++ b/pkg/engine/common.go @@ -7,8 +7,10 @@ import ( "path/filepath" "strings" + "github.com/containers/fetchit/pkg/engine/utils" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" + "k8s.io/klog/v2" ) type CommonMethod struct { @@ -70,6 +72,11 @@ func getDirectory(target *Target) string { return filepath.Base(trimDir) } +type empty struct { +} + +var BadCommitList map[string]map[string]map[plumbing.Hash]empty = make(map[string]map[string]map[plumbing.Hash]empty) + func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[]string) error { directory := getDirectory(target) if target.disconnected { @@ -79,6 +86,7 @@ func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[ localDevicePull(directory, target.device, "", false) } } + latest, err := getLatest(target) if err != nil { return fmt.Errorf("Failed to get latest commit: %v", err) @@ -89,10 +97,43 @@ func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[ return fmt.Errorf("Failed to get current commit: %v", err) } + if target.rollback && target.trackBadCommits { + if _, ok := BadCommitList[directory]; !ok { + BadCommitList[directory] = make(map[string]map[plumbing.Hash]empty) + } + + if _, ok := BadCommitList[directory][m.GetKind()]; !ok { + BadCommitList[directory][m.GetKind()] = make(map[plumbing.Hash]empty) + } + + if _, ok := BadCommitList[directory][m.GetKind()][latest]; ok { + klog.Infof("No changes applied to target %s this run, %s currently at %s", directory, m.GetKind(), current) + return nil + } + } + if latest != current { - if err := m.Apply(ctx, conn, current, latest, tag); err != nil { - return fmt.Errorf("Failed to apply changes: %v", err) + if err = m.Apply(ctx, conn, current, latest, tag); err != nil { + if target.rollback { + // Roll back automatically + klog.Errorf("Failed to apply changes, rolling back to %v: %v", current, err) + if err = checkout(target, current); err != nil { + return utils.WrapErr(err, "Failed to checkout %s", current) + } + if err = m.Apply(ctx, conn, latest, current, tag); err != nil { + // Roll back failed + return fmt.Errorf("Roll back failed, state between %s and %s: %v", current, latest, err) + } + if target.trackBadCommits { + BadCommitList[directory][m.GetKind()][latest] = empty{} + } + return fmt.Errorf("Rolled back to %v: %v", current, err) + } else { + return fmt.Errorf("Failed to apply changes from %v to %v: %v", current, latest, err) + } + } + updateCurrent(ctx, target, latest, m.GetKind(), m.GetName()) logger.Infof("Moved %s from %s to %s for git target %s", m.GetName(), current.String()[:hashReportLen], latest, target.url) } else { diff --git a/pkg/engine/types.go b/pkg/engine/types.go index 347ae28a..2ba05ed0 100644 --- a/pkg/engine/types.go +++ b/pkg/engine/types.go @@ -36,6 +36,8 @@ type TargetConfig struct { Device string `mapstructure:"device"` Disconnected bool `mapstructure:"disconnected"` VerifyCommitsInfo *VerifyCommitsInfo `mapstructure:"verifyCommitsInfo"` + TrackBadCommits bool `mapstructure:"trackBadCommits"` + Rollback bool `mapstructure:"rollback"` Branch string `mapstructure:"branch"` Ansible []*Ansible `mapstructure:"ansible"` FileTransfer []*FileTransfer `mapstructure:"filetransfer"` @@ -58,6 +60,8 @@ type Target struct { disconnected bool gitsignVerify bool gitsignRekorURL string + trackBadCommits bool + rollback bool } type SchedInfo struct {