Attempt on optimizing the builds

doc/source/quickstart.rst

@@ -133,7 +133,7 @@ First, install an API platform to target. **The recommended *target* API
 level is 27**, you can replace it with a different number but
 keep in mind other API versions are less well-tested and older devices
 are still supported down to the **recommended specified *minimum*
-API/NDK API level 21**::
+API/NDK API level 24**::
 
   $SDK_DIR/tools/bin/sdkmanager "platforms;android-27"
 

pythonforandroid/recipe.py

@@ -531,11 +531,10 @@ def unpack(self, arch):
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         """Return the env specialized for the recipe
         """
-        if arch is None:
-            arch = self.filtered_archs[0]
+        arch = arch or self.filtered_archs[0]
         env = arch.get_env(with_flags_in_cc=with_flags_in_cc)
 
-        for proxy_key in ['HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY', 'https_proxy']:
+        for proxy_key in ('HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY', 'https_proxy'):
             if proxy_key in environ:
                 env[proxy_key] = environ[proxy_key]
 

pythonforandroid/recipes/python3/__init__.py

@@ -73,58 +73,72 @@ class Python3Recipe(TargetPythonRecipe):
     configure_args = [
         '--host={android_host}',
         '--build={android_build}',
-        '--enable-shared',
         '--enable-ipv6',
         '--enable-loadable-sqlite-extensions',
-        '--without-static-libpython',
-        '--without-readline',
+        '--enable-shared',
+
+        # Attempt on making the builds lighter
+        '--disable-test-modules',
+        '--without-c-locale-coercion',
+        '--without-decimal-contextvar',
+        '--without-doc-strings',
         '--without-ensurepip',
+        '--without-readline',
+        '--without-static-libpython',
 
         # Android prefix
         '--prefix={prefix}',
-        '--enable-loadable-sqlite-extensions',
 
         # Special cross compile args
-        'ac_cv_file__dev_ptmx=yes',
-        'ac_cv_file__dev_ptc=no',
-        'ac_cv_header_sys_eventfd_h=no',
-        'ac_cv_little_endian_double=yes',
         'ac_cv_header_bzlib_h=no',
+        'ac_cv_header_sys_eventfd_h=no',
+        'py_cv_module__curses=n/a',
+        'py_cv_module__curses_panel=n/a',
+        'py_cv_module__tkinter=n/a'
     ]
 
     '''The configure arguments needed to build the python recipe. Those are
     used in method :meth:`build_arch` (if not overwritten like python3's
     recipe does).
     '''
 
-    MIN_NDK_API = 21
+    MIN_NDK_API = 24
     '''Sets the minimal ndk api number needed to use the recipe.
 
-    .. warning:: This recipe can be built only against API 21+, so it means
-        that any class which inherits from class:`GuestPythonRecipe` will have
-        this limitation.
+    .. warning:: Starting from Python 3.14 this recipe can only be built
+       against API 24+, so it means that any class which inherits from
+       class:`GuestPythonRecipe` will have this limitation.
     '''
 
     stdlib_dir_blacklist = {
         '__pycache__',
-        'test',
-        'tests',
-        'lib2to3',
+        'curses',
         'ensurepip',
         'idlelib',
+        'lib2to3',
+        'msilib',
+        'multiprocessing',
+        'pydoc_data',
+        'test',
+        'tests',
         'tkinter',
+        'turtledemo',
+        'venv'
     }
     '''The directories that we want to omit for our python bundle'''
 
     stdlib_filen_blacklist = [
-        '*.py',
         '*.exe',
+        '*.py',
         '*.whl',
+        'turtle.pyc'
     ]
     '''The file extensions that we want to blacklist for our python bundle'''
 
     site_packages_dir_blacklist = {
         '__pycache__',
+        '*.dist-info',
+        'bin',
         'tests'
     }
     '''The directories from site packages dir that we don't want to be included
@@ -139,7 +153,8 @@ class Python3Recipe(TargetPythonRecipe):
     if the full path contains any of these exceptions.'''
 
     site_packages_filen_blacklist = [
-        '*.py'
+        '*.py',
+        '*.pyx'
     ]
     '''The file extensions from site packages dir that we don't want to be
     included in our python bundle.'''
@@ -235,19 +250,20 @@ def prebuild_arch(self, arch):
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         env = super().get_recipe_env(arch)
         env['HOSTARCH'] = arch.command_prefix
-
         env['CC'] = arch.get_clang_exe(with_target=True)
-
-        env['PATH'] = (
-            '{hostpython_dir}:{old_path}').format(
-                hostpython_dir=self.get_recipe(
-                    'host' + self.name, self.ctx).get_path_to_python(),
-                old_path=env['PATH'])
-
+        env['PATH'] = '{hostpython_dir}:{old_path}'.format(
+            hostpython_dir=self.get_recipe(
+                'host' + self.name, self.ctx
+            ).get_path_to_python(),
+            old_path=env['PATH']
+        )
         env['CFLAGS'] = ' '.join(
             [
+                '-ffunction-sections',
+                '-fdata-sections',
                 '-fPIC',
-                '-DANDROID'
+                '-Oz',
+                '-g0'
             ]
         )
 
@@ -256,6 +272,8 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
             # Note: The -L. is to fix a bug in python 3.7.
             # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
             env['LDFLAGS'] += ' -L. -fuse-ld=lld'
+            env['LDFLAGS'] += ' -Wl,--gc-sections'
+            env['LDFLAGS'] += ' -Wl,--strip-all'
         else:
             warning('lld not found, linking without it. '
                     'Consider installing lld if linker errors occur.')
@@ -402,7 +420,7 @@ def compile_python_files(self, dir):
             longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
         '''
         args = [self.ctx.hostpython]
-        args += ['-OO', '-m', 'compileall', '-b', '-f', dir]
+        args += ['-OO', '-m', 'compileall', '-b', '-f', '-q', dir]
         subprocess.call(args)
 
     def create_python_bundle(self, dirn, arch):

pythonforandroid/recommendations.py

@@ -162,7 +162,7 @@ def check_target_api(api, arch):
         warning(OLD_API_MESSAGE)
 
 
-MIN_NDK_API = 21
+MIN_NDK_API = 24
 RECOMMENDED_NDK_API = 24
 OLD_NDK_API_MESSAGE = ('NDK API less than {} is not supported'.format(MIN_NDK_API))
 TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE = (

pythonforandroid/util.py

@@ -2,16 +2,16 @@
 from unittest import mock
 from fnmatch import fnmatch
 import logging
-from os.path import exists, join
-from os import getcwd, chdir, makedirs, walk
+from os.path import exists
+from os import getcwd, chdir, makedirs
 from pathlib import Path
 from platform import uname
 import shutil
 from tempfile import mkdtemp
 
 import packaging.version
 
-from pythonforandroid.logger import (logger, Err_Fore, error, info)
+from pythonforandroid.logger import logger, Err_Fore, error, info
 
 LOGGER = logging.getLogger("p4a.util")
 
@@ -66,23 +66,33 @@ def walk_valid_filens(base_dir, invalid_dir_names, invalid_file_patterns, exclud
     """
 
     excluded_dir_exceptions = [] if excluded_dir_exceptions is None else excluded_dir_exceptions
-
-    for dirn, subdirs, filens in walk(base_dir):
-        allow_invalid_dirs = any(ex in dirn for ex in excluded_dir_exceptions)
-
-        # Remove invalid subdirs so that they will not be walked
-        if not allow_invalid_dirs:
-            for i in reversed(range(len(subdirs))):
-                subdir = subdirs[i]
-                if subdir in invalid_dir_names:
-                    subdirs.pop(i)
-
-        for filen in filens:
-            for pattern in invalid_file_patterns:
-                if fnmatch(filen, pattern):
-                    break
-            else:
-                yield join(dirn, filen)
+    base_dir = Path(base_dir)
+
+    for path in base_dir.glob("**/*"):
+        if path.is_dir():
+            continue
+
+        rel_parts = path.relative_to(base_dir).parts[:-1]
+        cum_path = base_dir
+        skip = False
+        for part in rel_parts:
+            allow_invalid_dirs = any(ex in str(cum_path) for ex
+                                     in excluded_dir_exceptions)
+            if not allow_invalid_dirs and any(
+                fnmatch(part, pattern) for pattern in invalid_dir_names
+            ):
+                skip = True
+                break
+            cum_path = cum_path / part
+
+        if skip:
+            continue
+
+        if any(fnmatch(path.name, pattern) for pattern
+               in invalid_file_patterns):
+            continue
+
+        yield str(path)
 
 
 def load_source(module, filename):

testapps/setup_testapp_python3_sqlite_openssl.py

@@ -2,7 +2,7 @@
 
 options = {'apk': {'requirements': 'requests,peewee,sdl2,pyjnius,kivy,python3',
                    'android-api': 36,
-                   'ndk-api': 21,
+                   'ndk-api': 24,
                    'bootstrap': 'sdl2',
                    'dist-name': 'bdisttest_python3_sqlite_openssl_googlendk',
                    'ndk-version': '10.3.2',

testapps/setup_vispy.py

@@ -4,7 +4,7 @@
                    'requirements': 'python3,vispy',
                    'blacklist-requirements': 'openssl,sqlite3',
                    'android-api': 33,
-                   'ndk-api': 21,
+                   'ndk-api': 24,
                    'bootstrap': 'empty',
                    'ndk-dir': '/home/asandy/android/android-ndk-r17c',
                    'dist-name': 'bdisttest',

tests/recipes/test_python3.py

@@ -40,7 +40,7 @@ def test_compile_python_files(self, mock_subprocess):
         hostpy = self.recipe.ctx.hostpython = '/fake/hostpython3'
         self.recipe.compile_python_files(fake_compile_dir)
         mock_subprocess.assert_called_once_with(
-            [hostpy, '-OO', '-m', 'compileall', '-b', '-f', fake_compile_dir],
+            [hostpy, '-OO', '-m', 'compileall', '-b', '-f', '-q', fake_compile_dir],
         )
 
     @mock.patch("pythonforandroid.recipe.Recipe.check_recipe_choices")
@@ -61,7 +61,7 @@ def test_get_recipe_env(
         )
         env = self.recipe.get_recipe_env(self.arch)
 
-        self.assertIn('-fPIC -DANDROID', env["CFLAGS"])
+        self.assertIn('-fPIC', env["CFLAGS"])
         self.assertEqual(env["CC"], self.arch.get_clang_exe(with_target=True))
 
         # make sure that the mocked methods are actually called

tests/test_archs.py

@@ -50,8 +50,8 @@ class ArchSetUpBaseClass(object):
 
     def setUp(self):
         self.ctx = Context()
-        self.ctx.ndk_api = 21
-        self.ctx.android_api = 27
+        self.ctx.ndk_api = 24
+        self.ctx.android_api = 33
         self.ctx._sdk_dir = "/opt/android/android-sdk"
         self.ctx._ndk_dir = "/opt/android/android-ndk"
         self.ctx.ndk = AndroidNDK(self.ctx._ndk_dir)

tests/test_bootstrap.py

@@ -30,8 +30,8 @@ class BaseClassSetupBootstrap:
     def setUp(self):
         Recipe.recipes = {}  # clear Recipe class cache
         self.ctx = Context()
-        self.ctx.ndk_api = 21
-        self.ctx.android_api = 27
+        self.ctx.ndk_api = 24
+        self.ctx.android_api = 33
         self.ctx._sdk_dir = "/opt/android/android-sdk"
         self.ctx._ndk_dir = "/opt/android/android-ndk"
         self.ctx.ndk = AndroidNDK(self.ctx._ndk_dir)

tests/test_distribution.py

@@ -13,11 +13,11 @@
     "dist_name": "sdl2_dist",
     "bootstrap": "sdl2",
     "archs": ["armeabi", "armeabi-v7a", "x86", "x86_64", "arm64-v8a"],
-    "ndk_api": 21,
+    "ndk_api": 24,
     "use_setup_py": False,
     "recipes": ["hostpython3", "python3", "sdl2", "kivy", "requests"],
     "hostpython": "/some/fake/hostpython3",
-    "python_version": "3.7",
+    "python_version": "3.10",
 }
 
 
@@ -33,8 +33,8 @@ def setUp(self):
         """Configure a :class:`~pythonforandroid.build.Context` so we can
         perform our unittests"""
         self.ctx = Context()
-        self.ctx.ndk_api = 21
-        self.ctx.android_api = 27
+        self.ctx.ndk_api = 24
+        self.ctx.android_api = 33
         self.ctx._sdk_dir = "/opt/android/android-sdk"
         self.ctx._ndk_dir = "/opt/android/android-ndk"
         self.ctx.setup_dirs(os.getcwd())
@@ -162,7 +162,7 @@ def test_get_distributions(
         self.assertIsInstance(dists[0], Distribution)
         self.assertEqual(dists[0].name, "sdl2_dist")
         self.assertEqual(dists[0].dist_dir, "sdl2-python3")
-        self.assertEqual(dists[0].ndk_api, 21)
+        self.assertEqual(dists[0].ndk_api, 24)
         self.assertEqual(
             dists[0].recipes,
             ["hostpython3", "python3", "sdl2", "kivy", "requests"],