Backup rework

simplyblock_cli/cli-reference.yaml

@@ -2669,6 +2669,14 @@ commands:
             help: "The target storage node id."
             dest: node
             type: str
+          - name: "--access-key-id"
+            help: "Access key for the backup's bucket, when it is not this cluster's own."
+            dest: access_key_id
+            type: secret
+          - name: "--secret-access-key"
+            help: "Secret key for the backup's bucket, when it is not this cluster's own."
+            dest: secret_access_key
+            type: secret
       - name: export
         help: "Export backup metadata to a JSON file for cross-cluster restore."
         arguments:
@@ -2686,17 +2694,81 @@ commands:
             help: "The output file path."
             dest: output
             type: str
-      - name: import
-        help: "Import backup metadata from a JSON file."
+      - name: discover
+        help: "List the backups a bucket contains, reading its manifests. Needs no cluster."
         arguments:
-          - name: "metadata_file"
-            help: "The path to JSON metadata file."
-            dest: metadata_file
+          - name: "--bucket"
+            help: "The bucket holding the backups."
+            dest: bucket
+            type: str
+            required: true
+          - name: "--region"
+            help: "The bucket's region. Omit to let the AWS SDK resolve it."
+            dest: region
             type: str
+          - name: "--endpoint"
+            help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS."
+            dest: endpoint
+            type: str
+          - name: "--access-key-id"
+            help: "Access key for the bucket. Omit to use the node's instance role."
+            dest: access_key_id
+            type: secret
+          - name: "--secret-access-key"
+            help: "Secret key for the bucket. Omit to use the node's instance role."
+            dest: secret_access_key
+            type: secret
+          - name: "--no-verify-tls"
+            help: "Skip certificate verification for the endpoint."
+            dest: no_verify_tls
+            type: bool
+            action: store_true
+          - name: "--path-style"
+            help: "Use path-style addressing, as MinIO and most S3-compatible stores need."
+            dest: path_style
+            type: bool
+            action: store_true
+      - name: import
+        help: "Register backups into this cluster, from a bucket or from an exported file."
+        arguments:
           - name: "--cluster-id"
             help: "The target cluster to import into (required for cross-cluster restore)."
             dest: cluster_id
             type: str
+          - name: "--from-file"
+            help: "Path to a JSON file produced by 'backup export'. Mutually exclusive with --bucket."
+            dest: from_file
+            type: str
+          - name: "--bucket"
+            help: "Read manifests straight from this bucket. Mutually exclusive with --from-file."
+            dest: bucket
+            type: str
+          - name: "--region"
+            help: "The bucket's region. Omit to let the AWS SDK resolve it."
+            dest: region
+            type: str
+          - name: "--endpoint"
+            help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS."
+            dest: endpoint
+            type: str
+          - name: "--access-key-id"
+            help: "Access key for the bucket. Omit to use the node's instance role."
+            dest: access_key_id
+            type: secret
+          - name: "--secret-access-key"
+            help: "Secret key for the bucket. Omit to use the node's instance role."
+            dest: secret_access_key
+            type: secret
+          - name: "--no-verify-tls"
+            help: "Skip certificate verification for the endpoint."
+            dest: no_verify_tls
+            type: bool
+            action: store_true
+          - name: "--path-style"
+            help: "Use path-style addressing, as MinIO and most S3-compatible stores need."
+            dest: path_style
+            type: bool
+            action: store_true
       - name: policy-add
         help: "Create a new backup policy."
         arguments:
@@ -2770,24 +2842,6 @@ commands:
             help: "The target id (storage pool or logical volume id)."
             dest: target_id
             type: str
-      - name: source-list
-        help: "List backup sources (local and imported clusters)."
-        arguments:
-          - name: "--cluster-id"
-            help: "The cluster id."
-            dest: cluster_id
-            type: str
-      - name: source-switch
-        help: "Switch the active S3 backup source to a different cluster. Use 'local' or the local cluster id to switch back."
-        arguments:
-          - name: "source_cluster_id"
-            help: "The source cluster id or 'local'."
-            dest: source_cluster_id
-            type: str
-          - name: "--cluster-id"
-            help: "The cluster id."
-            dest: cluster_id
-            type: str
   - name: "qos"
     help: "QoS Commands"
     weight: 700

simplyblock_cli/cli.py

@@ -1083,14 +1083,13 @@ def init_backup(self):
         self.init_backup__delete(subparser)
         self.init_backup__restore(subparser)
         self.init_backup__export(subparser)
+        self.init_backup__discover(subparser)
         self.init_backup__import(subparser)
         self.init_backup__policy_add(subparser)
         self.init_backup__policy_remove(subparser)
         self.init_backup__policy_list(subparser)
         self.init_backup__policy_attach(subparser)
         self.init_backup__policy_detach(subparser)
-        self.init_backup__source_list(subparser)
-        self.init_backup__source_switch(subparser)
 
 
     def init_backup__list(self, subparser):
@@ -1107,17 +1106,36 @@ def init_backup__restore(self, subparser):
         subcommand.add_argument('--lvol', help='The new logical volume name.', type=str, dest='lvol_name', required=True)
         subcommand.add_argument('--pool', help='The target pool name or id.', type=str, dest='pool', required=True)
         subcommand.add_argument('--node', help='The target storage node id.', type=str, dest='node')
+        subcommand.add_argument('--access-key-id', help='Access key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='access_key_id')
+        subcommand.add_argument('--secret-access-key', help='Secret key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='secret_access_key')
 
     def init_backup__export(self, subparser):
         subcommand = self.add_sub_command(subparser, 'export', 'Export backup metadata to a JSON file for cross-cluster restore.')
         subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id')
         subcommand.add_argument('--lvol', help='Filter exports to a specific logical volume name.', type=str, dest='lvol_name')
         subcommand.add_argument('-o', '--output', help='The output file path.', type=str, dest='output')
 
+    def init_backup__discover(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'discover', 'List the backups a bucket contains, reading its manifests. Needs no cluster.')
+        subcommand.add_argument('--bucket', help='The bucket holding the backups.', type=str, dest='bucket', required=True)
+        subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region')
+        subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint')
+        subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id')
+        subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key')
+        subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true')
+        subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true')
+
     def init_backup__import(self, subparser):
-        subcommand = self.add_sub_command(subparser, 'import', 'Import backup metadata from a JSON file.')
-        subcommand.add_argument('metadata_file', help='The path to JSON metadata file.', type=str)
+        subcommand = self.add_sub_command(subparser, 'import', 'Register backups into this cluster, from a bucket or from an exported file.')
         subcommand.add_argument('--cluster-id', help='The target cluster to import into (required for cross-cluster restore).', type=str, dest='cluster_id')
+        subcommand.add_argument('--from-file', help='Path to a JSON file produced by \'backup export\'. Mutually exclusive with --bucket.', type=str, dest='from_file')
+        subcommand.add_argument('--bucket', help='Read manifests straight from this bucket. Mutually exclusive with --from-file.', type=str, dest='bucket')
+        subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region')
+        subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint')
+        subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id')
+        subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key')
+        subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true')
+        subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true')
 
     def init_backup__policy_add(self, subparser):
         subcommand = self.add_sub_command(subparser, 'policy-add', 'Create a new backup policy.')
@@ -1147,15 +1165,6 @@ def init_backup__policy_detach(self, subparser):
         subcommand.add_argument('target_type', help='The target type.', type=str, choices=['pool','lvol',])
         subcommand.add_argument('target_id', help='The target id (storage pool or logical volume id).', type=str)
 
-    def init_backup__source_list(self, subparser):
-        subcommand = self.add_sub_command(subparser, 'source-list', 'List backup sources (local and imported clusters).')
-        subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id')
-
-    def init_backup__source_switch(self, subparser):
-        subcommand = self.add_sub_command(subparser, 'source-switch', 'Switch the active S3 backup source to a different cluster. Use \'local\' or the local cluster id to switch back.')
-        subcommand.add_argument('source_cluster_id', help='The source cluster id or \'local\'.', type=str)
-        subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id')
-
 
     def init_qos(self):
         subparser = self.add_command('qos', 'QoS Commands')
@@ -1652,6 +1661,8 @@ def run(self):
                     ret = self.backup__restore(sub_command, args)
                 elif sub_command in ['export']:
                     ret = self.backup__export(sub_command, args)
+                elif sub_command in ['discover']:
+                    ret = self.backup__discover(sub_command, args)
                 elif sub_command in ['import']:
                     ret = self.backup__import(sub_command, args)
                 elif sub_command in ['policy-add']:
@@ -1664,10 +1675,6 @@ def run(self):
                     ret = self.backup__policy_attach(sub_command, args)
                 elif sub_command in ['policy-detach']:
                     ret = self.backup__policy_detach(sub_command, args)
-                elif sub_command in ['source-list']:
-                    ret = self.backup__source_list(sub_command, args)
-                elif sub_command in ['source-switch']:
-                    ret = self.backup__source_switch(sub_command, args)
                 else:
                     self.parser.print_help()