feat: atomic cascades, SCA challenge recipients, exact rate limits and card fixes

.github/workflows/build_container.yml

@@ -240,17 +240,20 @@ jobs:
             test_filter: >-
               code.api.v4_0_0.Dynamic
           - shard: 10
-            name: "PropGatedPublicEndpoint (read_*_requires_role=true)"
+            name: "PropGatedPublicEndpoint (read_*_requires_role=true, allow_cascade_delete=false)"
             # JsonSchemaValidationPublicPropTrueTest / AuthenticationTypeValidationPublicPropTrueTest
             # need read_json_schema_validation_requires_role /
             # read_authentication_type_validation_requires_role forced true, which is baked into
             # Http4s400's ResourceDoc error list at object-init time -- a value only one JVM can
             # observe. Every other shard boots with these props unset (false, the default) and
             # the pom.xml default tagsToExclude skips this tag there. This shard is the only place
             # the true branch is exercised; see the "Run tests" step's shard-10 special case below.
+            # CascadeDeleteDisabledTest is here for the opposite reason: every other shard has
+            # allow_cascade_delete=true, and only this JVM starts with it false.
             test_filter: >-
               code.api.v4_0_0.JsonSchemaValidationPublicPropTrueTest
               code.api.v4_0_0.AuthenticationTypeValidationPublicPropTrueTest
+              code.api.v4_0_0.CascadeDeleteDisabledTest
 
     services:
       redis:
@@ -341,6 +344,7 @@ jobs:
           echo allow_sandbox_data_import=true >> obp-api/src/main/resources/props/test.default.props
           echo sandbox_data_import_secret=change_me >> obp-api/src/main/resources/props/test.default.props
           echo allow_account_deletion=true >> obp-api/src/main/resources/props/test.default.props
+          echo allow_cascade_delete=true >> obp-api/src/main/resources/props/test.default.props
           echo allowed_internal_redirect_urls = /,/oauth/authorize >> obp-api/src/main/resources/props/test.default.props
           echo transactionRequests_enabled=true >> obp-api/src/main/resources/props/test.default.props
           echo transactionRequests_supported_types=SEPA,SANDBOX_TAN,FREE_FORM,COUNTERPARTY,ACCOUNT,SIMPLE >> obp-api/src/main/resources/props/test.default.props
@@ -449,6 +453,8 @@ jobs:
           if [ "${{ matrix.shard }}" = "10" ]; then
             export OBP_READ_JSON_SCHEMA_VALIDATION_REQUIRES_ROLE="true"
             export OBP_READ_AUTHENTICATION_TYPE_VALIDATION_REQUIRES_ROLE="true"
+            # The cascade-delete routes are fixed when the API objects initialise; off here only.
+            export OBP_ALLOW_CASCADE_DELETE="false"
             EXTRA_MVN_ARGS="-DtagsToInclude=PropGatedPublicEndpoint -Dtest.tagsToExclude="
           fi
 

.github/workflows/build_pull_request.yml

@@ -234,17 +234,20 @@ jobs:
             test_filter: >-
               code.api.v4_0_0.Dynamic
           - shard: 10
-            name: "PropGatedPublicEndpoint (read_*_requires_role=true)"
+            name: "PropGatedPublicEndpoint (read_*_requires_role=true, allow_cascade_delete=false)"
             # JsonSchemaValidationPublicPropTrueTest / AuthenticationTypeValidationPublicPropTrueTest
             # need read_json_schema_validation_requires_role /
             # read_authentication_type_validation_requires_role forced true, which is baked into
             # Http4s400's ResourceDoc error list at object-init time -- a value only one JVM can
             # observe. Every other shard boots with these props unset (false, the default) and
             # the pom.xml default tagsToExclude skips this tag there. This shard is the only place
             # the true branch is exercised; see the "Run tests" step's shard-10 special case below.
+            # CascadeDeleteDisabledTest is here for the opposite reason: every other shard has
+            # allow_cascade_delete=true, and only this JVM starts with it false.
             test_filter: >-
               code.api.v4_0_0.JsonSchemaValidationPublicPropTrueTest
               code.api.v4_0_0.AuthenticationTypeValidationPublicPropTrueTest
+              code.api.v4_0_0.CascadeDeleteDisabledTest
 
     services:
       redis:
@@ -335,6 +338,7 @@ jobs:
           echo allow_sandbox_data_import=true >> obp-api/src/main/resources/props/test.default.props
           echo sandbox_data_import_secret=change_me >> obp-api/src/main/resources/props/test.default.props
           echo allow_account_deletion=true >> obp-api/src/main/resources/props/test.default.props
+          echo allow_cascade_delete=true >> obp-api/src/main/resources/props/test.default.props
           echo allowed_internal_redirect_urls = /,/oauth/authorize >> obp-api/src/main/resources/props/test.default.props
           echo transactionRequests_enabled=true >> obp-api/src/main/resources/props/test.default.props
           echo transactionRequests_supported_types=SEPA,SANDBOX_TAN,FREE_FORM,COUNTERPARTY,ACCOUNT,SIMPLE >> obp-api/src/main/resources/props/test.default.props
@@ -453,6 +457,8 @@ jobs:
           if [ "${{ matrix.shard }}" = "10" ]; then
             export OBP_READ_JSON_SCHEMA_VALIDATION_REQUIRES_ROLE="true"
             export OBP_READ_AUTHENTICATION_TYPE_VALIDATION_REQUIRES_ROLE="true"
+            # The cascade-delete routes are fixed when the API objects initialise; off here only.
+            export OBP_ALLOW_CASCADE_DELETE="false"
             EXTRA_MVN_ARGS="-DtagsToInclude=PropGatedPublicEndpoint -Dtest.tagsToExclude="
           fi
 

CLAUDE.md

@@ -295,9 +295,9 @@ Shards are defined per-matrix-entry in `.github/workflows/build_pull_request.yml
 | `code.connector`, `code.util`, `code.api.Authentication*`, `code.api.dauthTest`, `code.api.DirectLoginTest`, `code.api.gateWayloginTest`, `code.api.OBPRestHelperTest`, `code.entitlement`, `code.bankaccountcreation`, `code.bankconnectors`, `code.container`, `code.management`, `code.metrics`, `code.concurrency` | 8 |
 | anything else | **8** (catch-all) |
 | `code.api.v4_0_0.Dynamic*` | 9 |
-| `code.api.v4_0_0.JsonSchemaValidationPublicPropTrueTest`, `code.api.v4_0_0.AuthenticationTypeValidationPublicPropTrueTest` (tagged `PropGatedPublicEndpoint`) | 10 |
+| `code.api.v4_0_0.JsonSchemaValidationPublicPropTrueTest`, `code.api.v4_0_0.AuthenticationTypeValidationPublicPropTrueTest`, `code.api.v4_0_0.CascadeDeleteDisabledTest` (tagged `PropGatedPublicEndpoint`) | 10 |
 
-Shard 10 is a special case: it's the only shard that overrides pom.xml's default `tagsToExclude` (which otherwise skips `PropGatedPublicEndpoint` everywhere) and sets `OBP_READ_JSON_SCHEMA_VALIDATION_REQUIRES_ROLE`/`OBP_READ_AUTHENTICATION_TYPE_VALIDATION_REQUIRES_ROLE=true`, because those two props are baked into `Http4s400`'s `ResourceDoc` error lists at object-init time — a single JVM can only ever observe one value of each, so the `true` branch needs its own shard while every other shard (which boots with the props unset, i.e. `false`) exercises the default branch. `run_tests_parallel.sh` (local runner) mirrors this with a dedicated sequential step after its 4 main shards — see that script's own comment near `PropGatedPublicEndpoint`.
+Shard 10 is a special case: it's the only shard that overrides pom.xml's default `tagsToExclude` (which otherwise skips `PropGatedPublicEndpoint` everywhere) and sets `OBP_READ_JSON_SCHEMA_VALIDATION_REQUIRES_ROLE`/`OBP_READ_AUTHENTICATION_TYPE_VALIDATION_REQUIRES_ROLE=true`, because those two props are baked into `Http4s400`'s `ResourceDoc` error lists at object-init time — a single JVM can only ever observe one value of each, so the `true` branch needs its own shard while every other shard (which boots with the props unset, i.e. `false`) exercises the default branch. Shard 10 also sets `OBP_ALLOW_CASCADE_DELETE=false` for `CascadeDeleteDisabledTest`: every other shard runs with `allow_cascade_delete=true`, and the cascade-delete routes are likewise fixed at object-init time. `run_tests_parallel.sh` (local runner) mirrors this with a dedicated sequential step after its 4 main shards — see that script's own comment near `PropGatedPublicEndpoint`.
 
 To explicitly move a package to a different shard, add it to that shard's `test_filter` block — it will be excluded from the catch-all automatically. `run_tests_parallel.sh` (local runner) uses a coarser 4-shard layout that folds all 9 CI shards' coverage into 4 wildcardSuites groups — see its own header comment for the mapping.
 

obp-api/pom.xml

@@ -60,7 +60,7 @@
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcpg-jdk18on</artifactId>
-      <version>1.84</version>
+      <version>1.85</version>
     </dependency>
     <dependency>
       <groupId>org.http4s</groupId>
@@ -75,7 +75,7 @@
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcpkix-jdk18on</artifactId>
-      <version>1.84</version>
+      <version>1.85</version>
     </dependency>
     <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
     <dependency>
@@ -633,7 +633,9 @@
                  observe. Excluded here by default (every ordinary shard/local/IDE run boots with
                  the props unset, i.e. false) and run separately with -Dtest.tagsToExclude=
                  -DtagsToInclude=PropGatedPublicEndpoint plus the two OBP_READ_*_REQUIRES_ROLE
-                 env vars set true (see run_tests_parallel.sh and the CI job that runs them);
+                 env vars set true (see run_tests_parallel.sh and the CI job that runs them).
+                 CascadeDeleteDisabledTest shares the tag and runs there with
+                 OBP_ALLOW_CASCADE_DELETE=false;
                  see the ${test.tagsToExclude} property declared above for why this is not a
                  plain literal. -->
             <tagsToExclude>${test.tagsToExclude}</tagsToExclude>

obp-api/src/main/resources/props/sample.props.template

@@ -639,6 +639,20 @@ messageQueue.createBankAccounts=true
 ## Set this to true if you want to allow users to delete accounts (local ones like HBCI connected)
 allow_account_deletion=true
 
+## Cascade-delete endpoints: each deletes a bank object together with everything stored under it.
+##   DELETE /obp/v4.0.0/management/cascading/banks/BANK_ID
+##   DELETE /obp/v4.0.0/management/cascading/banks/BANK_ID/accounts/ACCOUNT_ID
+##   DELETE /obp/v4.0.0/management/cascading/banks/BANK_ID/accounts/ACCOUNT_ID/transactions/TRANSACTION_ID
+##   DELETE /obp/v4.0.0/management/cascading/banks/BANK_ID/products/PRODUCT_CODE
+##   DELETE /obp/v4.0.0/management/cascading/banks/BANK_ID/customers/CUSTOMER_ID
+## When false (the default) they are not routed and not listed in the Resource Docs of any version,
+## so a request gets 404. Set to true to make them available. Read at startup. The Dynamic Entity
+## cascade deletes (.../dynamic-entities/cascade/DYNAMIC_ENTITY_ID) are not affected.
+## The Resource Docs responses are cached in Redis (staticResourceDocsObp.cache.ttl.seconds), so after a
+## change and restart they can list the old set of endpoints until that TTL passes, or until the rd_static
+## and rd_all cache namespaces are invalidated (POST /obp/v6.0.0/management/cache/namespaces/invalidate).
+allow_cascade_delete=false
+
 ## Secret key that allows access to api calls to get info about oauth tokens. You should change this
 ## to your own secret key
 BankMockKey=change_me
@@ -1237,6 +1251,7 @@ display_internal_errors=false
 # Please note that in case that redirect_url_value contains special word PLACEHOLDER it will be replaced with actual ID
 # http://127.0.0.1:8080/confirm-bg-consent-request?PAYMENT_ID=PLACEHOLDER
 # psu_make_payment_sca_redirect_url = redirect_url_value
+# When it is not set (or blank), the response to a payment initiation has no scaRedirect link.
 # -------------------------------------------------------------- Authentication methods --
 
 ## This property is used for documenting at Resource Doc. It may include the port also (but not /obp)

obp-api/src/main/resources/props/test.default.props.template

@@ -115,6 +115,9 @@ sandbox_data_import_secret=change_me
 
 #set this to true if you want to allow users to delete accounts
 allow_account_deletion=true
+# The cascade-delete endpoints are off by default; Delete*CascadeTest and DeleteCascadeCounterpartiesTest
+# need them. CascadeDeleteDisabledTest runs with them off.
+allow_cascade_delete=true
 
 # This needs to be a list all the types of transaction_requests that we have tests for. Else those tests will fail
 transactionRequests_supported_types=SANDBOX_TAN,COUNTERPARTY,SEPA,ACCOUNT_OTP,ACCOUNT,SIMPLE,AGENT_CASH_WITHDRAWAL,CARD

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -384,24 +384,12 @@ class Boot extends MdcLoggable {
       }
 
       // The remaining two stay opt-in: nothing in the code requires them to exist, so whether an
-      // instance has them is still the operator's call. Their permission sets ARE code-defined
-      // though, so where they do exist they are kept current the same way -- the prop decides
-      // existence, not whether the code is authoritative about what a view grants.
-      APIUtil.getPropsValue("additional_system_views") match {
-        case Full(value) =>
-          val additionalSystemViewsFromProps = value.split(",").map(_.trim).toList
-          val additionalSystemViews = List(
-            SYSTEM_READ_TRANSACTIONS_BERLIN_GROUP_VIEW_ID,
-            SYSTEM_INITIATE_PAYMENTS_BERLIN_GROUP_VIEW_ID
-          )
-          for {
-            systemView <- additionalSystemViewsFromProps
-            if additionalSystemViews.exists(_ == systemView)
-          } {
-            Views.views.vend.ensureSystemViewUpToDate(systemView)
-          }
-        case _ => // Do nothing
-      }
+      // instance has them is still the operator's call. (The mapped connector is the exception: it lists
+      // both in viewsToGenerate, so a refresh creates them and grants them to the holders of their accounts
+      // whether or not they are listed here.) Their permission sets ARE code-defined though, so wherever they
+      // exist, listed or not, they are kept current the same way, unless reconcile_permissions_at_boot is false --
+      // the prop decides existence, not whether the code is authoritative about what a view grants.
+      Views.views.vend.ensureOptInSystemViewsUpToDate(APIUtil.getPropsValue("additional_system_views"))
 
     }
 

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala

@@ -4310,8 +4310,8 @@ object SwaggerDefinitionsJSON {
   lazy val callLimitPostJsonV600 = CallLimitPostJsonV600(
     from_date = DateWithDayExampleObject,
     to_date = DateWithDayExampleObject,
-    api_version = Some(ApiVersion.v6_0_0.toString),
-    api_name = Some("getConsumerCallLimits"),
+    api_version = None,
+    api_name = None,
     bank_id = None,
     per_second_call_limit = "100",
     per_minute_call_limit = "1000",

obp-api/src/main/scala/code/api/berlin/group/v1_3/JSONFactory_BERLIN_GROUP_1_3.scala

@@ -318,7 +318,7 @@ object JSONFactory_BERLIN_GROUP_1_3 extends CustomJsonFormats with MdcLoggable{
   )
 
   case class InitiatePaymentResponseLinks(
-    scaRedirect: LinkHrefJson,
+    scaRedirect: Option[LinkHrefJson],
     self: LinkHrefJson,
     status: LinkHrefJson,
     scaStatus: LinkHrefJson
@@ -801,18 +801,18 @@ object JSONFactory_BERLIN_GROUP_1_3 extends CustomJsonFormats with MdcLoggable{
 //      Remark: This code may be
     //map OBP transactionRequestId to BerlinGroup PaymentId
     val paymentId = transactionRequest.id.value
-    val scaRedirectUrlPattern = getPropsValue("psu_make_payment_sca_redirect_url")
-      .openOr(MissingPropsValueAtThisInstance + "psu_make_payment_sca_redirect_url")
-    val scaRedirectUrl =
-      if (scaRedirectUrlPattern.contains("PLACEHOLDER"))
-        scaRedirectUrlPattern.replace("PLACEHOLDER", paymentId)
+    // Without psu_make_payment_sca_redirect_url there is no page to send the PSU to, so the link is left out.
+    val scaRedirectUrl = getPropsValue("psu_make_payment_sca_redirect_url").toOption.filter(_.trim.nonEmpty).map { pattern =>
+      if (pattern.contains("PLACEHOLDER"))
+        pattern.replace("PLACEHOLDER", paymentId)
       else
-        s"$scaRedirectUrlPattern/${paymentId}"
+        s"$pattern/$paymentId"
+    }
     InitiatePaymentResponseJson(
       transactionStatus = mapTransactionStatus(transactionRequest.status),
       paymentId = paymentId,
       _links = InitiatePaymentResponseLinks(
-        scaRedirect = LinkHrefJson(s"$scaRedirectUrl/$paymentId"),
+        scaRedirect = scaRedirectUrl.map(LinkHrefJson(_)),
         self = LinkHrefJson(s"/${ConstantsBG.berlinGroupVersion1.apiShortVersion}/payments/sepa-credit-transfers/$paymentId"),
         status = LinkHrefJson(s"/${ConstantsBG.berlinGroupVersion1.apiShortVersion}/payments/$paymentId/status"),
         scaStatus = LinkHrefJson(s"/${ConstantsBG.berlinGroupVersion1.apiShortVersion}/payments/$paymentId/authorisations/${paymentId}")

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -2655,6 +2655,13 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
 
   def getDisabledEndpointOperationIds() : List[String] = APIUtil.getPropsValue("api_disabled_endpoints").getOrElse("").replace("[", "").replace("]", "").split(",").toList.filter(_.nonEmpty)
 
+  /**
+   * allow_cascade_delete (default false). The v4.0.0 /management/cascading/... endpoints are
+   * routed and documented only when this is true. It is read when the API version objects are
+   * initialised, so a change takes effect on restart. The Dynamic Entity cascade deletes, which
+   * back up to a ZZ_BAK_ entity first, are not affected.
+   */
+  def cascadeDeleteEnabled: Boolean = getPropsAsBoolValue("allow_cascade_delete", false)
 
 
   def getEnabledVersions() : List[String] = APIUtil.getPropsValue("api_enabled_versions").getOrElse("").replace("[", "").replace("]", "").split(",").toList.filter(_.nonEmpty).map(_.trim)

obp-api/src/main/scala/code/api/util/ApiRole.scala

@@ -1690,10 +1690,22 @@ object ApiRole extends MdcLoggable{
     }
   }
 
+  // Checked only by the /management/cascading endpoints, so not offered while allow_cascade_delete=false
+  // leaves those endpoints out. valueOf still resolves them, for entitlements granted earlier.
+  private lazy val cascadeDeleteRoleNames: Set[String] = Set[ApiRole](
+    canDeleteTransactionCascade,
+    canDeleteAccountCascade,
+    canDeleteBankCascade,
+    canDeleteProductCascade,
+    canDeleteCustomerCascade
+  ).map(_.toString)
+  private lazy val offeredRoleNames: List[String] =
+    roles.map(_.toString).filter(name => APIUtil.cascadeDeleteEnabled || !cascadeDeleteRoleNames.contains(name))
+
   def availableRoles: List[String] = {
     import scala.jdk.CollectionConverters._
     val dynamicRoles = dynamicApiRoles.keys().asScala.toList
-    dynamicRoles ::: roles.map(_.toString)
+    dynamicRoles ::: offeredRoleNames
   }
 
 }

obp-api/src/main/scala/code/api/util/ConsentUtil.scala

@@ -1057,6 +1057,35 @@ object Consent extends MdcLoggable {
     consentJwtBox.foreach(releaseVrpMandateArtefacts(consent, _))
   }
 
+  /**
+   * The consent statuses after which an SCA challenge raised for the consent can no longer be used:
+   * a REVOKED consent is re-authenticated with a fresh challenge, and REJECTED and EXPIRED are terminal.
+   */
+  val consentChallengeEndingStatuses: Set[ConsentStatus] =
+    Set(ConsentStatus.REVOKED, ConsentStatus.REJECTED, ConsentStatus.EXPIRED)
+
+  /**
+   * Delete the OBP_CONSENT_CHALLENGE rows of a consent once none of them can be used any more: when
+   * one has been answered correctly, or when the consent reaches one of consentChallengeEndingStatuses.
+   *
+   * Nothing reads these rows after the answer -- the attempt counter only matters while a challenge
+   * is still open -- so the row is not an audit trail, it is a hashed one-time password waiting to
+   * be swept. Berlin Group consent challenges are left alone (their authorisation endpoints keep
+   * listing them), and so are transaction request challenges.
+   *
+   * Never fails the caller: by the time this runs the answer or the status change has happened, and
+   * a row left behind is logged rather than turned into an error for something that succeeded.
+   */
+  def removeConsentChallenges(consentId: String): Unit =
+    code.transactionChallenge.Challenges.ChallengeProvider.vend
+      .deleteChallengesByConsentId(consentId, com.openbankproject.commons.model.enums.ChallengeType.OBP_CONSENT_CHALLENGE.toString) match {
+      case Failure(msg, ex, _) =>
+        logger.error(
+          s"removeConsentChallenges: the challenges of consent $consentId could not be deleted: $msg" +
+          ex.map(e => s" (${e.getClass.getSimpleName}: ${e.getMessage})").getOrElse(""))
+      case _ =>
+    }
+
   /**
    * Release the artefacts a VRP mandate created, when the consent that owns them is revoked.
    *