ENT-12129: Added full hosts delete and 2FA support to distributed cleanup script

templates/federated_reporting/distributed_cleanup.py

@@ -31,7 +31,7 @@
 import subprocess
 import sys
 from getpass import getpass
-from nova_api import NovaApi
+from nova_api import NovaApi, Unauthenticated, Unauthenticated2FA
 from cfsecret import read_secret, write_secret
 
 WORKDIR = None
@@ -65,26 +65,37 @@
 
 
 def interactive_setup_feeder(hub, email, fr_distributed_cleanup_password, force_interactive=False):
+    feeder_hostname = hub["ui_name"]
+    feeder_admin_user = input("Enter admin username for {} [admin]: ".format(feeder_hostname)) or "admin"
     if force_interactive:
         feeder_credentials = input(
-            "admin credentials for {}: ".format(
-                hub["ui_name"]
-            )
+            "admin credentials for {}: ".format(feeder_hostname)
         )
         print() # output newline for easier reading
     else:
         feeder_credentials = getpass(
-            prompt="Enter admin credentials for {}: ".format(
-                hub["ui_name"]
-            )
+            prompt="Enter admin password for {}: ".format(feeder_hostname)
         )
-    feeder_hostname = hub["ui_name"]
     feeder_api = NovaApi(
-        api_user="admin",
+        api_user=feeder_admin_user,
         api_password=feeder_credentials,
         cert_path=CERT_PATH,
         hostname=feeder_hostname,
     )
+    try:
+        feeder_api.status()
+    except Unauthenticated2FA:
+        token = getpass(prompt="Enter 2FA code for {}: ".format(feeder_hostname))
+        feeder_api = NovaApi(
+            api_user=feeder_admin_user,
+            api_password=feeder_credentials,
+            cert_path=CERT_PATH,
+            hostname=feeder_hostname,
+            two_factor_token=token,
+        )
+    except Unauthenticated:
+        print("admin credentials for {} are incorrect, try again".format(feeder_hostname))
+        sys.exit(1)
 
     logger.info("Creating fr_distributed_cleanup role on %s", feeder_hostname)
     response = feeder_api.put(
@@ -103,7 +114,7 @@ def interactive_setup_feeder(hub, email, fr_distributed_cleanup_password, force_
         )
         sys.exit(1)
     response = feeder_api.put_role_permissions(
-        "fr_distributed_cleanup", ["host.delete"]
+        "fr_distributed_cleanup", ["host.delete", "hosts-delete-permanently.delete"]
     )
     if response["status"] != 201:
         print("Unable to set RBAC permissions on role fr_distributed_cleanup")
@@ -130,6 +141,7 @@ def interactive_setup_feeder(hub, email, fr_distributed_cleanup_password, force_
 
 def interactive_setup(force_interactive=False):
     fr_distributed_cleanup_password = "".join(random.choices(string.digits + string.ascii_letters, k=20))
+    admin_user = input("Enter admin username for superhub {} [admin]: ".format(socket.getfqdn())) or "admin"
     if force_interactive:
         admin_pass = input("admin password for superhub {}: ".format(socket.getfqdn()))
         print() # newline for easier reading
@@ -138,27 +150,26 @@ def interactive_setup(force_interactive=False):
             prompt="Enter admin password for superhub {}: ".format(socket.getfqdn())
         )
 
-    api = NovaApi(api_user="admin", api_password=admin_pass)
+    api = NovaApi(api_user=admin_user, api_password=admin_pass)
 
     # first confirm that this host is a superhub
-    status = api.fr_hub_status()
-    if (
-        status["status"] == 200
-        and status["role"] == "superhub"
-        and status["configured"]
-    ):
-        logger.debug("This host is a superhub configured for Federated Reporting.")
-    else:
-        if status["status"] == 401:
-            print("admin credentials are incorrect, try again")
-            sys.exit(1)
-        else:
-            print(
-                "Check the status to ensure role is superhub and configured is True. {}".format(
-                    status
-                )
+    try:
+        status = api.fr_hub_status()
+    except Unauthenticated2FA:
+        token = getpass(prompt="Enter 2FA code for superhub {}: ".format(socket.getfqdn()))
+        api = NovaApi(api_user=admin_user, api_password=admin_pass, two_factor_token=token)
+        status = api.fr_hub_status()
+    except Unauthenticated:
+        print("admin credentials are incorrect, try again")
+        sys.exit(1)
+    if not (status["status"] == 200 and status["role"] == "superhub" and status["configured"]):
+        print(
+            "Check the status to ensure role is superhub and configured is True. {}".format(
+                status
             )
-            sys.exit(1)
+        )
+        sys.exit(1)
+    logger.debug("This host is a superhub configured for Federated Reporting.")
 
     feederResponse = api.fr_remote_hubs()
     if not feederResponse["hubs"]:
@@ -295,19 +306,26 @@ def main():
         )
         try:
             response = feeder_api.status()
-        except Exception as e:
-            print("Could not connect to {}, error: {}".format(feeder_hostname, e));
-            sys.exit(1);
-        if response["status"] == 401 and sys.stdout.isatty():
-            # auth error when running interactively
-            # assume it's a new feeder and offer to set it up interactively
-            hub_user = api.get( "user", "fr_distributed_cleanup")
-            if hub_user is None or 'email' not in hub_user:
-                email = 'fr_distributed_cleanup@{}'.format(hub['ui_name'])
+        except Unauthenticated2FA:
+            if sys.stdout.isatty():
+                hub_user = api.get("user", "fr_distributed_cleanup")
+                email = hub_user['email'] if hub_user and 'email' in hub_user else 'fr_distributed_cleanup@{}'.format(hub['ui_name'])
+                interactive_setup_feeder(hub, email, fr_distributed_cleanup_password)
+            else:
+                print("2FA required for feeder {}. Skipping".format(feeder_hostname))
+                continue
+        except Unauthenticated:
+            if sys.stdout.isatty():
+                hub_user = api.get("user", "fr_distributed_cleanup")
+                email = hub_user['email'] if hub_user and 'email' in hub_user else 'fr_distributed_cleanup@{}'.format(hub['ui_name'])
+                interactive_setup_feeder(hub, email, fr_distributed_cleanup_password)
             else:
-                email = hub_user['email']
-            interactive_setup_feeder(hub, email, fr_distributed_cleanup_password)
-        elif response["status"] != 200:
+                print("Unable to authenticate to feeder {}. Skipping".format(feeder_hostname))
+                continue
+        except Exception as e:
+            print("Could not connect to {}, error: {}".format(feeder_hostname, e))
+            sys.exit(1)
+        if response["status"] != 200:
             print(
                 "Unable to get status for feeder {}. Skipping".format(feeder_hostname)
             )
@@ -372,9 +390,11 @@ def main():
             # We only selected hostkey so will take the first value.
             host_to_delete = row[0]
 
-            response = feeder_api.delete("host", host_to_delete)
-            # both 202 Accepted and 404 Not Found are acceptable responses
-            if response["status"] not in [202, 404]:
+
+            responseDelete = feeder_api.delete("host", host_to_delete)
+            responsePermanentlyDelete = feeder_api.delete("hosts/delete-permanently", host_to_delete)
+            # both 202 Accepted/204 No Content and 200 Ok/404 Not Found are acceptable responses
+            if responseDelete["status"] not in [202, 204] or responsePermanentlyDelete["status"] not in [200, 404]:
                 logger.warning(
                     "Delete %s on feeder %s got %s status code",
                     host_to_delete,
@@ -393,17 +413,11 @@ def main():
             )
             continue
 
-        # simulate the host api delete process by setting current_timestamp in deleted column
-        # and delete from all federated tables similar to the clear_hosts_references() pgplsql function.
-        post_sql += "INSERT INTO __hosts (hostkey,deleted) VALUES"
-        deletes = []
+        # delete from __hosts and all federated tables similar to the clear_hosts_references() pgplsql function.
+        hostkeys_to_delete = []
         for hostkey in post_hostkeys:
-            deletes.append("('{}', CURRENT_TIMESTAMP)".format(hostkey))
-
-        delete_sql = ", ".join(deletes)
-        delete_sql += (
-            " ON CONFLICT (hostkey,hub_id) DO UPDATE SET deleted = excluded.deleted;\n"
-        )
+            hostkeys_to_delete.append("'{}'".format(hostkey))
+        delete_sql = "DELETE FROM __hosts WHERE hostkey IN ({});\n".format(",".join(hostkeys_to_delete))
         clear_sql = "set schema 'public';\n"
         for table in CFE_FR_TABLES:
             # special case of partitioning, operating on parent table will work

templates/federated_reporting/nova_api.py

@@ -34,6 +34,14 @@
 _DEFAULT_SECRETS_PATH = "{}/httpd/secrets.ini".format(_WORKDIR)
 
 
+class Unauthenticated(Exception):
+    pass
+
+
+class Unauthenticated2FA(Unauthenticated):
+    pass
+
+
 class NovaApi:
     def __init__(
         self,
@@ -42,6 +50,7 @@ def __init__(
         api_password=None,
         cert_path=None,
         ca_cert_dir=None,
+        two_factor_token=None,
     ):
         self._hostname = hostname or str(socket.getfqdn())
         self._api_user = api_user
@@ -69,6 +78,8 @@ def __init__(
             basic_auth="{}:{}".format(self._api_user, self._api_password)
         )
         self._headers["Content-Type"] = "application/json"
+        if two_factor_token:
+            self._headers["Cf-2fa-Token"] = two_factor_token
         # urllib3 v2.0 removed SubjectAltNameWarning and instead throws an error if no SubjectAltName is present in a certificate
         if hasattr(urllib3.exceptions, "SubjectAltNameWarning"):
             # if urllib3 is < v2.0 then SubjectAltNameWarning will exist and should be silenced
@@ -111,10 +122,17 @@ def _build_response(self, response):
             if not message:
                 if response.status == 201:
                     message = "Created"
+            if response.status == 401:
+                if "Invalid two-factor" in message:
+                    raise Unauthenticated2FA(message)
+                raise Unauthenticated(message)
             value["message"] = message
             value["status"] = response.status
         else:
-            data = json.loads(response.data.decode("utf-8"))
+            body = response.data.decode("utf-8").strip()
+            if not body:
+                return {"status": response.status}
+            data = json.loads(body)
             # some APIs like query API return a top-level data key which we want to skip for ease of use
             if "data" in data:
                 # data response e.g. query API returns top-level key 'data'