bugfix/use Reflections instead of ClassUtil for better Fat JAR class scanning

obp-api/pom.xml

@@ -271,11 +271,11 @@
       <version>4.0.3</version>
     </dependency>
     <!-- ********** flexmark END ********** -->
-    <!--scala utils, for type scan-->
+    <!--class scanning, replaces classutil for better Fat JAR support-->
     <dependency>
-      <groupId>org.clapper</groupId>
-      <artifactId>classutil_${scala.version}</artifactId>
-      <version>1.5.1</version>
+      <groupId>org.reflections</groupId>
+      <artifactId>reflections</artifactId>
+      <version>0.10.2</version>
     </dependency>
     <dependency>
       <groupId>com.github.grumlimited</groupId>

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -41,17 +41,9 @@ import code.api.ResourceDocs1_4_0.ResourceDocs300.{ResourceDocs310, ResourceDocs
 import code.api.ResourceDocs1_4_0._
 import code.api._
 import code.api.attributedefinition.AttributeDefinition
-import code.api.berlin.group.v1_3.{OBP_BERLIN_GROUP_1_3, OBP_BERLIN_GROUP_1_3_Alias}
 import code.api.berlin.group.ConstantsBG
-import code.api.STET.v1_4.OBP_STET_1_4
-import code.api.Polish.v2_1_1_1.OBP_PAPI_2_1_1_1
-import code.api.MxOF.{OBP_MXOF_1_0_0, CNBV9_1_0_0}
-import code.api.BahrainOBF.v1_0_0.{ApiCollector => BahrainApiCollector}
-import code.api.AUOpenBanking.v1_0_0.{ApiCollector => AUApiCollector}
-import code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200
-import code.api.UKOpenBanking.v3_1_0.OBP_UKOpenBanking_310
 import code.api.cache.Redis
-import code.api.util.APIUtil.{enableVersionIfAllowed, versionIsAllowed,errorJsonResponse, getPropsValue}
+import code.api.util.APIUtil.{enableVersionIfAllowed, errorJsonResponse, getPropsValue}
 import code.api.util.ApiRole._
 import code.api.util.ErrorMessages.MandatoryPropertyIsNotSet
 import code.api.util._
@@ -471,95 +463,7 @@ class Boot extends MdcLoggable {
     ApiVersion.setUrlPrefix(ApiPathZero)
 
     // Add the various API versions
-    val scannedApisCount = ScannedApis.versionMapScannedApis.size
-    logger.info(s"ClassScanUtils found $scannedApisCount ScannedApis implementations")
-
     ScannedApis.versionMapScannedApis.keys.foreach(enableVersionIfAllowed) // process all scanned apis versions
-
-
-    // Manual registration for ScannedApis if not already registered by ClassScanUtils
-    // This ensures all APIs work in Fat JAR environment where class scanning fails
-
-    if (!ScannedApis.versionMapScannedApis.contains(ConstantsBG.berlinGroupVersion1)) {
-      logger.warn("BGv1.3 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ConstantsBG.berlinGroupVersion1)) {
-        LiftRules.statelessDispatch.append(OBP_BERLIN_GROUP_1_3)
-        logger.info(s"${ConstantsBG.berlinGroupVersion1.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(OBP_BERLIN_GROUP_1_3_Alias.apiVersion)) {
-      logger.warn("BGv1.3 Alias was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(OBP_BERLIN_GROUP_1_3_Alias.apiVersion)) {
-        LiftRules.statelessDispatch.append(OBP_BERLIN_GROUP_1_3_Alias)
-        logger.info(s"${OBP_BERLIN_GROUP_1_3_Alias.apiVersion.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(ApiVersion.stetV14)) {
-      logger.warn("STET v1.4 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ApiVersion.stetV14)) {
-        LiftRules.statelessDispatch.append(OBP_STET_1_4)
-        logger.info(s"${ApiVersion.stetV14.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(ApiVersion.polishApiV2111)) {
-      logger.warn("Polish API v2.1.1.1 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ApiVersion.polishApiV2111)) {
-        LiftRules.statelessDispatch.append(OBP_PAPI_2_1_1_1)
-        logger.info(s"${ApiVersion.polishApiV2111.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(ApiVersion.mxofV100)) {
-      logger.warn("Mexico Open Finance v1.0.0 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ApiVersion.mxofV100)) {
-        LiftRules.statelessDispatch.append(OBP_MXOF_1_0_0)
-        logger.info(s"${ApiVersion.mxofV100.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(ApiVersion.cnbv9)) {
-      logger.warn("Mexico CNBV9 v1.0.0 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ApiVersion.cnbv9)) {
-        LiftRules.statelessDispatch.append(CNBV9_1_0_0)
-        logger.info(s"${ApiVersion.cnbv9.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(ApiVersion.bahrainObfV100)) {
-      logger.warn("Bahrain OBF v1.0.0 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ApiVersion.bahrainObfV100)) {
-        LiftRules.statelessDispatch.append(BahrainApiCollector)
-        logger.info(s"${ApiVersion.bahrainObfV100.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(ApiVersion.cdsAuV100)) {
-      logger.warn("Australia CDS v1.0.0 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ApiVersion.cdsAuV100)) {
-        LiftRules.statelessDispatch.append(AUApiCollector)
-        logger.info(s"${ApiVersion.cdsAuV100.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(ApiVersion.ukOpenBankingV20)) {
-      logger.warn("UK Open Banking v2.0.0 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ApiVersion.ukOpenBankingV20)) {
-        LiftRules.statelessDispatch.append(OBP_UKOpenBanking_200)
-        logger.info(s"${ApiVersion.ukOpenBankingV20.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
-    if (!ScannedApis.versionMapScannedApis.contains(ApiVersion.ukOpenBankingV31)) {
-      logger.warn("UK Open Banking v3.1.0 was NOT found by ClassScanUtils, registering manually")
-      if (versionIsAllowed(ApiVersion.ukOpenBankingV31)) {
-        LiftRules.statelessDispatch.append(OBP_UKOpenBanking_310)
-        logger.info(s"${ApiVersion.ukOpenBankingV31.fullyQualifiedVersion} was ENABLED (manual registration)")
-      }
-    }
-
     enableVersionIfAllowed(ApiVersion.v1_2_1)
     enableVersionIfAllowed(ApiVersion.v1_3_0)
     enableVersionIfAllowed(ApiVersion.v1_4_0)

obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala

@@ -7,7 +7,7 @@ import code.api.berlin.group.ConstantsBG
 object ApiVersionUtils {
 
   val scannedApis = ScannedApis.versionMapScannedApis.keysIterator.toList
-  val versions =
+  val versions = (
       v1_2_1 ::
       v1_3_0 ::
       v1_4_0 ::
@@ -25,6 +25,7 @@ object ApiVersionUtils {
       `dynamic-entity` ::
       ConstantsBG.berlinGroupVersion2 ::
       scannedApis
+    ).distinct
 
   def valueOf(value: String): ScannedApiVersion = {
 

obp-api/src/main/scala/code/util/ClassScanUtils.scala

@@ -1,114 +1,113 @@
 package code.util
 
-import java.io.File
-
-import com.openbankproject.commons.model.Bank
 import code.util.Helper.MdcLoggable
 import org.apache.commons.lang3.StringUtils
-import org.clapper.classutil.{ClassFinder, ClassInfo}
+import org.reflections.Reflections
+import org.reflections.scanners.Scanners
+import org.reflections.util.{ClasspathHelper, ConfigurationBuilder}
 import com.openbankproject.commons.util.ReflectUtils
 
+import scala.jdk.CollectionConverters._
 import scala.reflect.runtime.universe.TypeTag
 
 /**
-  * this is some util method to scan any class according some rules
+  * Utility methods to scan classes using Reflections library.
+  * Replaces classutil (org.clapper) which does not support Fat JAR environments.
   * @author shuang
   */
 object ClassScanUtils extends MdcLoggable {
 
-  lazy val finder = ClassFinder(getClassPath(this.getClass, classOf[Bank], classOf[String]))
+  // Scan the "code" package only to avoid scanning all dependencies
+  lazy val reflections: Reflections = {
+    val config = new ConfigurationBuilder()
+      .setUrls(ClasspathHelper.forPackage("code"))
+      .setScanners(Scanners.SubTypes.filterResultsBy(_ => true))
+    new Reflections(config)
+  }
 
   /**
     * get companion object or singleton object by class name
     * @param name object class name
     * @tparam U expect type
     * @return companion object or singleton object
     */
-  def companion[U:TypeTag](name : String) : U = {
-    val className = if(name.endsWith("$")) name else name + "$"
+  def companion[U: TypeTag](name: String): U = {
+    val className = if (name.endsWith("$")) name else name + "$"
     Class.forName(className).getDeclaredField("MODULE$").get(null).asInstanceOf[U]
   }
 
   /**
     * scan classpath to get all companion objects or singleton objects those implements given trait
     * @tparam T the trait type parameter
-    * @return all companion objects or singleton object those implements given clazz
+    * @return all companion objects or singleton objects those implement the given trait
     */
-  def getSubTypeObjects[T:TypeTag]: List[T] = {
+  def getSubTypeObjects[T: TypeTag]: List[T] = {
     val clazz = ReflectUtils.typeTagToClass[T]
-    val classes = try {
-      val allClasses = finder.getClasses().toList
-      logger.info(s"ClassScanUtils successfully scanned ${allClasses.size} classes from classpath")
-      allClasses
+    try {
+      val subTypes = reflections.getSubTypesOf(clazz).asScala.toList
+      logger.info(s"ClassScanUtils (Reflections) found ${subTypes.size} subtypes of ${clazz.getName}")
+      // companion objects have a class name ending with "$"
+      val objects = subTypes
+        .filter(c => c.getName.endsWith("$"))
+        .flatMap { c =>
+          try { Some(companion[T](c.getName)) }
+          catch { case e: Exception =>
+            logger.warn(s"Failed to load companion object ${c.getName}: ${e.getMessage}")
+            None
+          }
+        }
+      logger.info(s"Found ${objects.size} companion objects implementing ${clazz.getName}")
+      objects
     } catch {
-      case e: UnsupportedOperationException =>
-        // ASM version is too old for some class files (e.g. requires ASM7). In that case,
-        // skip scanned APIs instead of failing the whole application.
-        logger.warn(s"Class scanning failed with UnsupportedOperationException: ${e.getMessage}")
-        logger.warn("This is expected when running from a Fat JAR. Scanned APIs will not be auto-registered.")
-        Seq.empty
       case e: Exception =>
-        logger.warn(s"Class scanning failed with ${e.getClass.getSimpleName}: ${e.getMessage}")
-        Seq.empty
+        logger.warn(s"ClassScanUtils (Reflections) failed for ${clazz.getName}: ${e.getMessage}")
+        Nil
     }
-    val filtered = classes.filter(_.implements(clazz.getName))
-    logger.info(s"Found ${filtered.size} classes implementing ${clazz.getName}")
-    filtered.map(_.name).map(companion[T](_)).toList
   }
 
   /**
-    * find all fit classes, do filter with predict function
-    * @param predict check whether include this type in the result
-    * @return all fit type names
+    * find all fit classes, filtered by a predicate on the Class object.
+    * @param predict check whether to include this class in the result
+    * @return all matching class names (without trailing "$")
     */
-  def findTypes(predict: ClassInfo => Boolean): List[String] = {
-    val classes = try {
-      finder.getClasses().toList
+  def findTypes(predict: Class[_] => Boolean): List[String] = {
+    try {
+      // getSubTypesOf(Object) returns all known classes in the scanned packages
+      reflections.getSubTypesOf(classOf[Object]).asScala.toList
+        .filter { c =>
+          try { predict(c) }
+          catch { case _: Exception => false }
+        }
+        .map { c =>
+          val name = c.getName
+          if (name.endsWith("$")) name.substring(0, name.length - 1) else name
+        }
     } catch {
-      case _: UnsupportedOperationException =>
-        Seq.empty
+      case e: Exception =>
+        logger.warn(s"ClassScanUtils.findTypes failed: ${e.getMessage}")
+        Nil
     }
-    classes
-      .filter(predict)
-      .map(it => {
-        val name = it.name
-        if(name.endsWith("$")) name.substring(0, name.length - 1)
-        else name
-      }) //some companion type name ends with $, it added by scalac, should remove from class name
-      .toList
   }
 
   /**
-    * get given class exists jar Files
-    * @param classes to find class paths contains these class files
-    * @return this class exists jar File
-    */
-  private[this] def getClassPath(classes: Class[_]*): Seq[File] =  classes.map { clazz =>
-      val classFile = "/" + clazz.getName.replace('.', '/') + ".class"
-      val uri = clazz.getResource(classFile).toURI.toString
-      val path = uri.replaceFirst("^(jar:|file:){0,2}(.*?)\\!?\\Q" + classFile + "\\E$", "$2")
-      new File(path)
-    }
-
-  /**
-    * get all subtype of net.liftweb.mapper.LongKeyedMapper, so we can register scanned db models dynamic
+    * get all subtype of net.liftweb.mapper.LongKeyedMapper, so we can register scanned db models dynamically
     * @param packageName scanned root package name
-    * @return all scanned ClassInfo
+    * @return all matching class names
     */
-  def getMappers(packageName:String = ""): Seq[ClassInfo] = {
-    val mapperInterface = "net.liftweb.mapper.LongKeyedMapper"
-    val classes = try {
-      finder.getClasses().toList
+  def getMappers(packageName: String = ""): Seq[String] = {
+    try {
+      val mapperInterface = Class.forName("net.liftweb.mapper.LongKeyedMapper")
+      val all = reflections.getSubTypesOf(mapperInterface).asScala.toSeq
+        .map(_.getName)
+      if (StringUtils.isNotBlank(packageName))
+        all.filter(_.startsWith(packageName))
+      else
+        all
     } catch {
-      case _: UnsupportedOperationException =>
-        Seq.empty
-    }
-    val infos = classes.filter(it => it.interfaces.contains(mapperInterface))
-    if(StringUtils.isNoneBlank()) {
-      infos.filter(classInfo => classInfo.name.startsWith(packageName))
-    } else {
-      infos
+      case e: Exception =>
+        logger.warn(s"ClassScanUtils.getMappers failed: ${e.getMessage}")
+        Nil
     }
   }
 
-}
+}

obp-api/src/test/scala/code/util/MappedClassNameTest.scala

@@ -120,11 +120,11 @@ class MappedClassNameTest extends FeatureSpec {
       "code.CustomerDependants.MappedCustomerDependant",
     )
 
-  val newMappedTypes = ClassScanUtils.findTypes{ info =>
-    val typeName = info.name
+  val newMappedTypes = ClassScanUtils.findTypes{ clazz =>
+    val typeName = clazz.getName
     !typeName.endsWith("$") &&
       !oldMappedTypeNames.contains(typeName) &&
-      mapperClazz.isAssignableFrom(Class.forName(typeName, false, mapperClazz.getClassLoader))
+      mapperClazz.isAssignableFrom(clazz)
   }.toSet
   feature("Validate New Entity name and column name") {