diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 1d5e3e9104..de075be4ee 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -531,11 +531,10 @@ def unpack(self, arch): def get_recipe_env(self, arch=None, with_flags_in_cc=True): """Return the env specialized for the recipe """ - if arch is None: - arch = self.filtered_archs[0] + arch = arch or self.filtered_archs[0] env = arch.get_env(with_flags_in_cc=with_flags_in_cc) - for proxy_key in ['HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY', 'https_proxy']: + for proxy_key in ('HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY', 'https_proxy'): if proxy_key in environ: env[proxy_key] = environ[proxy_key] diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 3da132d698..bdaae71c48 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -73,23 +73,28 @@ class Python3Recipe(TargetPythonRecipe): configure_args = [ '--host={android_host}', '--build={android_build}', - '--enable-shared', '--enable-ipv6', '--enable-loadable-sqlite-extensions', - '--without-static-libpython', - '--without-readline', + '--enable-shared', + + # Attempt on making the builds lighter + '--disable-test-modules', + '--without-c-locale-coercion', + '--without-decimal-contextvar', + '--without-doc-strings', '--without-ensurepip', + '--without-readline', + '--without-static-libpython', # Android prefix '--prefix={prefix}', - '--enable-loadable-sqlite-extensions', # Special cross compile args - 'ac_cv_file__dev_ptmx=yes', - 'ac_cv_file__dev_ptc=no', - 'ac_cv_header_sys_eventfd_h=no', - 'ac_cv_little_endian_double=yes', 'ac_cv_header_bzlib_h=no', + 'ac_cv_header_sys_eventfd_h=no', + 'py_cv_module__curses=n/a', + 'py_cv_module__curses_panel=n/a', + 'py_cv_module__tkinter=n/a' ] '''The configure arguments needed to build the python recipe. Those are @@ -100,32 +105,43 @@ class Python3Recipe(TargetPythonRecipe): MIN_NDK_API = 21 '''Sets the minimal ndk api number needed to use the recipe. - .. warning:: This recipe can be built only against API 21+, so it means - that any class which inherits from class:`GuestPythonRecipe` will have - this limitation. + .. warning:: Starting from Python 3.14 this recipe can only be built + against API 21+, so it means that any class which inherits from + class:`GuestPythonRecipe` will have this limitation. ''' stdlib_dir_blacklist = { '__pycache__', - 'test', - 'tests', - 'lib2to3', + 'curses', 'ensurepip', 'idlelib', + 'lib2to3', + 'msilib', + 'multiprocessing', + 'pydoc_data', + 'test', + 'tests', 'tkinter', + 'turtledemo', + 'venv' } '''The directories that we want to omit for our python bundle''' stdlib_filen_blacklist = [ - '*.py', '*.exe', + '*.py', '*.whl', + 'turtle.pyc' ] '''The file extensions that we want to blacklist for our python bundle''' site_packages_dir_blacklist = { '__pycache__', - 'tests' + '*.dist-info', + 'bin', + 'tests', + 'setuptools', + '_distutils_hack' } '''The directories from site packages dir that we don't want to be included in our python bundle.''' @@ -139,7 +155,8 @@ class Python3Recipe(TargetPythonRecipe): if the full path contains any of these exceptions.''' site_packages_filen_blacklist = [ - '*.py' + '*.py', + '*.pyx' ] '''The file extensions from site packages dir that we don't want to be included in our python bundle.''' @@ -235,19 +252,18 @@ def prebuild_arch(self, arch): def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch) env['HOSTARCH'] = arch.command_prefix - env['CC'] = arch.get_clang_exe(with_target=True) - - env['PATH'] = ( - '{hostpython_dir}:{old_path}').format( - hostpython_dir=self.get_recipe( - 'host' + self.name, self.ctx).get_path_to_python(), - old_path=env['PATH']) - + env['PATH'] = '{hostpython_dir}:{old_path}'.format( + hostpython_dir=self.get_recipe( + 'host' + self.name, self.ctx + ).get_path_to_python(), + old_path=env['PATH'] + ) env['CFLAGS'] = ' '.join( [ - '-fPIC', - '-DANDROID' + '-ffunction-sections', + '-fdata-sections', + '-fPIC' ] ) @@ -402,7 +418,7 @@ def compile_python_files(self, dir): longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488) ''' args = [self.ctx.hostpython] - args += ['-OO', '-m', 'compileall', '-b', '-f', dir] + args += ['-OO', '-m', 'compileall', '-b', '-f', '-q', dir] subprocess.call(args) def create_python_bundle(self, dirn, arch): diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index 9b1752beea..75dd97e534 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -2,8 +2,8 @@ from unittest import mock from fnmatch import fnmatch import logging -from os.path import exists, join -from os import getcwd, chdir, makedirs, walk +from os.path import exists +from os import getcwd, chdir, makedirs from pathlib import Path from platform import uname import shutil @@ -11,7 +11,7 @@ import packaging.version -from pythonforandroid.logger import (logger, Err_Fore, error, info) +from pythonforandroid.logger import logger, Err_Fore, error, info LOGGER = logging.getLogger("p4a.util") @@ -66,23 +66,33 @@ def walk_valid_filens(base_dir, invalid_dir_names, invalid_file_patterns, exclud """ excluded_dir_exceptions = [] if excluded_dir_exceptions is None else excluded_dir_exceptions - - for dirn, subdirs, filens in walk(base_dir): - allow_invalid_dirs = any(ex in dirn for ex in excluded_dir_exceptions) - - # Remove invalid subdirs so that they will not be walked - if not allow_invalid_dirs: - for i in reversed(range(len(subdirs))): - subdir = subdirs[i] - if subdir in invalid_dir_names: - subdirs.pop(i) - - for filen in filens: - for pattern in invalid_file_patterns: - if fnmatch(filen, pattern): - break - else: - yield join(dirn, filen) + base_dir = Path(base_dir) + + for path in base_dir.glob("**/*"): + if path.is_dir(): + continue + + rel_parts = path.relative_to(base_dir).parts[:-1] + cum_path = base_dir + skip = False + for part in rel_parts: + allow_invalid_dirs = any(ex in str(cum_path) for ex + in excluded_dir_exceptions) + if not allow_invalid_dirs and any( + fnmatch(part, pattern) for pattern in invalid_dir_names + ): + skip = True + break + cum_path = cum_path / part + + if skip: + continue + + if any(fnmatch(path.name, pattern) for pattern + in invalid_file_patterns): + continue + + yield str(path) def load_source(module, filename): diff --git a/tests/recipes/test_python3.py b/tests/recipes/test_python3.py index e3b69e4bcb..b5e239cbcd 100644 --- a/tests/recipes/test_python3.py +++ b/tests/recipes/test_python3.py @@ -40,7 +40,7 @@ def test_compile_python_files(self, mock_subprocess): hostpy = self.recipe.ctx.hostpython = '/fake/hostpython3' self.recipe.compile_python_files(fake_compile_dir) mock_subprocess.assert_called_once_with( - [hostpy, '-OO', '-m', 'compileall', '-b', '-f', fake_compile_dir], + [hostpy, '-OO', '-m', 'compileall', '-b', '-f', '-q', fake_compile_dir], ) @mock.patch("pythonforandroid.recipe.Recipe.check_recipe_choices") @@ -61,7 +61,7 @@ def test_get_recipe_env( ) env = self.recipe.get_recipe_env(self.arch) - self.assertIn('-fPIC -DANDROID', env["CFLAGS"]) + self.assertIn('-fPIC', env["CFLAGS"]) self.assertEqual(env["CC"], self.arch.get_clang_exe(with_target=True)) # make sure that the mocked methods are actually called diff --git a/tests/test_util.py b/tests/test_util.py index 744e17132e..12940f9ce2 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -9,56 +9,56 @@ class TestUtil(unittest.TestCase): - """ + ''' An inherited class of `unittest.TestCase`to test the module :mod:`~pythonforandroid.util`. - """ + ''' - @mock.patch("pythonforandroid.util.makedirs") + @mock.patch('pythonforandroid.util.makedirs') def test_ensure_dir(self, mock_makedirs): - """ + ''' Basic test for method :meth:`~pythonforandroid.util.ensure_dir`. Here we make sure that the mentioned method is called only once. - """ - util.ensure_dir("fake_directory") - mock_makedirs.assert_called_once_with("fake_directory") + ''' + util.ensure_dir('fake_directory') + mock_makedirs.assert_called_once_with('fake_directory') - @mock.patch("shutil.rmtree") - @mock.patch("pythonforandroid.util.mkdtemp") + @mock.patch('shutil.rmtree') + @mock.patch('pythonforandroid.util.mkdtemp') def test_temp_directory(self, mock_mkdtemp, mock_shutil_rmtree): - """ + ''' Basic test for method :meth:`~pythonforandroid.util.temp_directory`. We perform this test by `mocking` the command `mkdtemp` and `shutil.rmtree` and we make sure that those functions are called in the proper place. - """ - mock_mkdtemp.return_value = "/temp/any_directory" + ''' + mock_mkdtemp.return_value = '/temp/any_directory' with util.temp_directory(): mock_mkdtemp.assert_called_once() mock_shutil_rmtree.assert_not_called() - mock_shutil_rmtree.assert_called_once_with("/temp/any_directory") + mock_shutil_rmtree.assert_called_once_with('/temp/any_directory') - @mock.patch("pythonforandroid.util.chdir") + @mock.patch('pythonforandroid.util.chdir') def test_current_directory(self, moch_chdir): - """ + ''' Basic test for method :meth:`~pythonforandroid.util.current_directory`. We `mock` chdir and we check that the command is executed once we are inside a python's `with` statement. Then we check that `chdir has been called with the proper arguments inside this `with` statement and also that, once we leave the `with` statement, is called again with the current working path. - """ - chdir_dir = "/temp/any_directory" + ''' + chdir_dir = '/temp/any_directory' # test chdir to existing directory with util.current_directory(chdir_dir): - moch_chdir.assert_called_once_with("/temp/any_directory") + moch_chdir.assert_called_once_with('/temp/any_directory') moch_chdir.assert_has_calls( - [mock.call("/temp/any_directory"), mock.call(os.getcwd())] + [mock.call('/temp/any_directory'), mock.call(os.getcwd())] ) def test_current_directory_exception(self): - """ + ''' Another test for method :meth:`~pythonforandroid.util.current_directory`, but here we check that using the method with a non-existing-directory raises an `OSError` @@ -67,15 +67,14 @@ def test_current_directory_exception(self): .. note:: test chdir to non-existing directory, should raise error, for py3 the exception is FileNotFoundError and IOError for py2, to avoid introduce conditions, we test with a more generic exception - """ + ''' with self.assertRaises(OSError), util.current_directory( - "/fake/directory" + '/fake/directory' ): pass - @mock.patch("pythonforandroid.util.walk") - def test_walk_valid_filens(self, mock_walk): - """ + def test_walk_valid_filens(self): + ''' Test method :meth:`~pythonforandroid.util.walk_valid_filens` In here we simulate the following directory structure: @@ -83,7 +82,7 @@ def test_walk_valid_filens(self, mock_walk): |-- README |-- setup.py |-- __pycache__ - |-- |__ + |-- |__ somefile |__Lib |-- abc.pyc |-- abc.py @@ -98,32 +97,45 @@ def test_walk_valid_filens(self, mock_walk): :emphasize-lines: 2-4 expected_result = { - "/fake_dir/README", - "/fake_dir/Lib/abc.pyc", - "/fake_dir/Lib/ctypes/util.pyc", + '/fake_dir/README', + '/fake_dir/Lib/abc.pyc', + '/fake_dir/Lib/ctypes/util.pyc', } - """ - simulated_walk_result = [ - ["/fake_dir", ["__pycache__", "Lib"], ["README", "setup.py"]], - ["/fake_dir/Lib", ["ctypes"], ["abc.pyc", "abc.py"]], - ["/fake_dir/Lib/ctypes", [], ["util.pyc", "util.py"]], - ] - mock_walk.return_value = simulated_walk_result - file_ens = util.walk_valid_filens( - "/fake_dir", ["__pycache__"], ["*.py"] - ) - self.assertIsInstance(file_ens, types.GeneratorType) - expected_result = { - "/fake_dir/README", - "/fake_dir/Lib/abc.pyc", - "/fake_dir/Lib/ctypes/util.pyc", - } - result = set(file_ens) + ''' + with TemporaryDirectory() as base_dir: + base_dir = Path(base_dir) - self.assertEqual(result, expected_result) + # Build the fake directory structure + (base_dir / 'README').touch() + (base_dir / 'setup.py').touch() + + (base_dir / '__pycache__').mkdir() + (base_dir / '__pycache__' / 'somefile').touch() + + (base_dir / 'Lib').mkdir() + (base_dir / 'Lib' / 'abc.pyc').touch() + (base_dir / 'Lib' / 'abc.py').touch() + + (base_dir / 'Lib' / 'ctypes').mkdir() + (base_dir / 'Lib' / 'ctypes' / 'util.pyc').touch() + (base_dir / 'Lib' / 'ctypes' / 'util.py').touch() + + file_ens = util.walk_valid_filens( + str(base_dir), ['__pycache__'], ['*.py'] + ) + self.assertIsInstance(file_ens, types.GeneratorType) + + expected_result = { + str(base_dir / 'README'), + str(base_dir / 'Lib' / 'abc.pyc'), + str(base_dir / 'Lib' / 'ctypes' / 'util.pyc'), + } + result = set(file_ens) + + self.assertEqual(result, expected_result) def test_util_exceptions(self): - """ + ''' Test exceptions for a couple of methods: - method :meth:`~pythonforandroid.util.BuildInterruptingException` @@ -133,23 +145,23 @@ def test_util_exceptions(self): :meth:`~pythonforandroid.util.BuildInterruptingException` and we run it inside method :meth:`~pythonforandroid.util.handle_build_exception` to make sure that it raises an `SystemExit`. - """ + ''' exc = util.BuildInterruptingException( - "missing dependency xxx", instructions="pip install --user xxx" + 'missing dependency xxx', instructions='pip install --user xxx' ) with self.assertRaises(SystemExit): util.handle_build_exception(exc) def test_move(self): with mock.patch( - "pythonforandroid.util.LOGGER" + 'pythonforandroid.util.LOGGER' ) as m_logger, TemporaryDirectory() as base_dir: - new_path = Path(base_dir) / "new" + new_path = Path(base_dir) / 'new' # Set up source - old_path = Path(base_dir) / "old" - with open(old_path, "w") as outfile: - outfile.write("Temporary content") + old_path = Path(base_dir) / 'old' + with open(old_path, 'w') as outfile: + outfile.write('Temporary content') # Non existent source with self.assertRaises(FileNotFoundError): @@ -169,12 +181,12 @@ def test_move(self): m_logger.reset_mock() # Move over existing: - existing_path = Path(base_dir) / "existing" + existing_path = Path(base_dir) / 'existing' existing_path.touch() util.move(new_path, existing_path) - with open(existing_path, "r") as infile: - assert infile.read() == "Temporary content" + with open(existing_path, 'r') as infile: + assert infile.read() == 'Temporary content' m_logger.debug.assert_called() m_logger.error.assert_not_called() m_logger.reset_mock() @@ -183,7 +195,7 @@ def test_touch(self): # Just checking the new file case. # Assume the existing file timestamp case will work if this does. with TemporaryDirectory() as base_dir: - new_file_path = Path(base_dir) / "new_file" + new_file_path = Path(base_dir) / 'new_file' assert not new_file_path.exists() util.touch(new_file_path) assert new_file_path.exists() @@ -191,21 +203,21 @@ def test_touch(self): def test_build_tools_version_sort_key(self): build_tools_versions = [ - "26.0.1", - "26.0.0", - "26.0.2", - "32.0.0 rc1", - "31.0.0", - "999something", + '26.0.1', + '26.0.0', + '26.0.2', + '32.0.0 rc1', + '31.0.0', + '999something', ] expected_result = [ - "999something", # invalid version - "26.0.0", - "26.0.1", - "26.0.2", - "31.0.0", - "32.0.0 rc1", + '999something', # invalid version + '26.0.0', + '26.0.1', + '26.0.2', + '31.0.0', + '32.0.0 rc1', ] result = sorted( @@ -217,83 +229,83 @@ def test_build_tools_version_sort_key(self): def test_max_build_tool_version(self): build_tools_versions = [ - "26.0.1", - "26.0.0", - "26.0.2", - "32.0.0 rc1", - "31.0.0", - "999something", + '26.0.1', + '26.0.0', + '26.0.2', + '32.0.0 rc1', + '31.0.0', + '999something', ] - expected_result = "32.0.0 rc1" + expected_result = '32.0.0 rc1' result = util.max_build_tool_version(build_tools_versions) self.assertEqual(result, expected_result) def test_load_source(self): - """ + ''' Test method :meth:`~pythonforandroid.util.load_source`. We test loading a Python module from a file path using importlib. - """ + ''' with TemporaryDirectory() as temp_dir: # Create a test module file - test_module_path = Path(temp_dir) / "test_module.py" - with open(test_module_path, "w") as f: - f.write("TEST_VALUE = 42\n") - f.write("def test_function():\n") + test_module_path = Path(temp_dir) / 'test_module.py' + with open(test_module_path, 'w') as f: + f.write('TEST_VALUE = 42\n') + f.write('def test_function():\n') f.write(" return 'hello'\n") # Load the module - loaded_module = util.load_source("test_module", str(test_module_path)) + loaded_module = util.load_source('test_module', str(test_module_path)) # Verify the module was loaded correctly self.assertEqual(loaded_module.TEST_VALUE, 42) self.assertEqual(loaded_module.test_function(), 'hello') - @mock.patch("pythonforandroid.util.exists") - @mock.patch("shutil.rmtree") + @mock.patch('pythonforandroid.util.exists') + @mock.patch('shutil.rmtree') def test_rmdir_exists(self, mock_rmtree, mock_exists): - """ + ''' Test method :meth:`~pythonforandroid.util.rmdir` when directory exists. We mock exists to return True and verify rmtree is called. - """ + ''' mock_exists.return_value = True - util.rmdir("/fake/directory") - mock_rmtree.assert_called_once_with("/fake/directory", False) + util.rmdir('/fake/directory') + mock_rmtree.assert_called_once_with('/fake/directory', False) - @mock.patch("pythonforandroid.util.exists") - @mock.patch("shutil.rmtree") + @mock.patch('pythonforandroid.util.exists') + @mock.patch('shutil.rmtree') def test_rmdir_not_exists(self, mock_rmtree, mock_exists): - """ + ''' Test method :meth:`~pythonforandroid.util.rmdir` when directory doesn't exist. We mock exists to return False and verify rmtree is not called. - """ + ''' mock_exists.return_value = False - util.rmdir("/fake/directory") + util.rmdir('/fake/directory') mock_rmtree.assert_not_called() - @mock.patch("pythonforandroid.util.exists") - @mock.patch("shutil.rmtree") + @mock.patch('pythonforandroid.util.exists') + @mock.patch('shutil.rmtree') def test_rmdir_ignore_errors(self, mock_rmtree, mock_exists): - """ + ''' Test method :meth:`~pythonforandroid.util.rmdir` with ignore_errors flag. We verify that the ignore_errors parameter is passed to rmtree. - """ + ''' mock_exists.return_value = True - util.rmdir("/fake/directory", ignore_errors=True) - mock_rmtree.assert_called_once_with("/fake/directory", True) + util.rmdir('/fake/directory', ignore_errors=True) + mock_rmtree.assert_called_once_with('/fake/directory', True) - @mock.patch("pythonforandroid.util.mock") + @mock.patch('pythonforandroid.util.mock') def test_patch_wheel_setuptools_logging(self, mock_mock): - """ + ''' Test method :meth:`~pythonforandroid.util.patch_wheel_setuptools_logging`. We verify it returns a mock.patch object for the wheel logging module. - """ + ''' mock_patch_obj = mock.Mock() mock_mock.patch.return_value = mock_patch_obj result = util.patch_wheel_setuptools_logging() - mock_mock.patch.assert_called_once_with("wheel._setuptools_logging.configure") + mock_mock.patch.assert_called_once_with('wheel._setuptools_logging.configure') self.assertEqual(result, mock_patch_obj)