Merge back commoncode with git history intact

pyproject-scancode-toolkit-mini.toml

@@ -56,7 +56,6 @@ dependencies = [
     "click >= 6.7, !=7.0, !=8.1.8;python_version<'3.10'",
     "click >= 8.2.0;python_version>='3.10'",
     "colorama >= 0.3.9",
-    "commoncode >= 32.4.2",
     "cyseq >= 0.1.2",
     "container-inspector >= 31.0.0",
     "debian-inspector >= 31.1.0",

pyproject-scancode-toolkit.toml

@@ -56,7 +56,6 @@ dependencies = [
     "click >= 6.7, !=7.0, !=8.1.8;python_version<'3.10'",
     "click >= 8.2.0;python_version>='3.10'",
     "colorama >= 0.3.9",
-    "commoncode >= 32.4.2",
     "cyseq >= 0.1.2",
     "container-inspector >= 31.0.0",
     "debian-inspector >= 31.1.0",

pyproject.toml

@@ -57,7 +57,6 @@ dependencies = [
     "click >= 6.7, !=7.0, !=8.1.8;python_version<'3.10'",
     "click >= 8.2.0;python_version>='3.10'",
     "colorama >= 0.3.9",
-    "commoncode >= 32.4.2",
     "cyseq >= 0.1.2",
     "container-inspector >= 31.0.0",
     "debian-inspector >= 31.1.0",

requirements.txt

@@ -11,7 +11,6 @@ chardet==5.2.0
 charset-normalizer==3.4.4
 click==8.3.1
 colorama==0.4.6
-commoncode==32.4.2
 construct==2.10.70
 container-inspector==33.0.0
 cryptography==46.0.5

src/commoncode/__init__.py

@@ -0,0 +1,29 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/commoncode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+def set_re_max_cache(max_cache=1000000):
+    """
+    Set re and fnmatch _MAXCACHE to 1Million items to cache compiled regex
+    aggressively. Their default is a maximum of 100 items and many utilities and
+    libraries use a lot of regexes: therefore 100 is not enough to benefit from
+    caching.
+    """
+    import fnmatch
+    import re
+
+    remax = getattr(re, "_MAXCACHE", 0)
+    if remax < max_cache:
+        setattr(re, "_MAXCACHE", max_cache)
+
+    fnmatchmax = getattr(fnmatch, "_MAXCACHE", 0)
+    if fnmatchmax < max_cache:
+        setattr(fnmatch, "_MAXCACHE", max_cache)
+
+
+set_re_max_cache()

src/commoncode/archive.py

@@ -0,0 +1,133 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/commoncode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import gzip
+import os
+import tarfile
+import zipfile
+from functools import partial
+from os import path
+
+from commoncode.system import on_windows
+from commoncode.system import py314
+
+"""
+Mimimal tar and zip file handling, primarily for testing.
+"""
+
+
+def _extract_tar_raw(test_path, target_dir, to_bytes, *args, **kwargs):
+    """
+    Raw simplified extract for certain really weird paths and file
+    names.
+    """
+    tar = None
+    try:
+        tar = tarfile.open(test_path)
+        tar.extractall(path=target_dir)
+    finally:
+        if tar:
+            tar.close()
+
+
+extract_tar_raw = partial(_extract_tar_raw, to_bytes=True)
+
+extract_tar_uni = partial(_extract_tar_raw, to_bytes=False)
+
+
+def extract_tar(location, target_dir, verbatim=False, filter=None, *args, **kwargs):
+    """
+    Extract a tar archive at location in the target_dir directory.
+    If `verbatim` is True preserve the permissions.
+    """
+    # always for using bytes for paths on all OSses... tar seems to use bytes internally
+    # and get confused otherwise
+    location = os.fsencode(location)
+    with open(location, "rb") as input_tar:
+        tar = None
+        try:
+            tar = tarfile.open(fileobj=input_tar)
+            tarinfos = tar.getmembers()
+            to_extract = []
+            for tarinfo in tarinfos:
+                if tar_can_extract(tarinfo, verbatim):
+                    if not verbatim:
+                        tarinfo.mode = 0o755
+                    to_extract.append(tarinfo)
+            if py314 and filter:
+                tar.extractall(target_dir, members=to_extract, filter=filter)
+            else:
+                tar.extractall(target_dir, members=to_extract)
+        finally:
+            if tar:
+                tar.close()
+
+
+def extract_zip(location, target_dir, *args, **kwargs):
+    """
+    Extract a zip archive file at location in the target_dir directory.
+    """
+    if not path.isfile(location) and zipfile.is_zipfile(location):
+        raise Exception("Incorrect zip file %(location)r" % locals())
+
+    with zipfile.ZipFile(location) as zipf:
+        for info in zipf.infolist():
+            name = info.filename
+            content = zipf.read(name)
+            target = path.join(target_dir, name)
+            if not path.exists(path.dirname(target)):
+                os.makedirs(path.dirname(target))
+            if not content and target.endswith(path.sep):
+                if not path.exists(target):
+                    os.makedirs(target)
+            if not path.exists(target):
+                with open(target, "wb") as f:
+                    f.write(content)
+
+
+def extract_zip_raw(location, target_dir, *args, **kwargs):
+    """
+    Extract a zip archive file at location in the target_dir directory.
+    Use the builtin extractall function
+    """
+    if not path.isfile(location) and zipfile.is_zipfile(location):
+        raise Exception("Incorrect zip file %(location)r" % locals())
+
+    with zipfile.ZipFile(location) as zipf:
+        zipf.extractall(path=target_dir)
+
+
+def tar_can_extract(tarinfo, verbatim):
+    """
+    Return True if a tar member can be extracted to handle OS specifics.
+    If verbatim is True, always return True.
+    """
+    if tarinfo.ischr():
+        # never extract char devices
+        return False
+
+    if verbatim:
+        # extract all on all OSse
+        return True
+
+    # FIXME: not sure hard links are working OK on Windows
+    include = tarinfo.type in tarfile.SUPPORTED_TYPES
+    exclude = tarinfo.isdev() or (on_windows and tarinfo.issym())
+
+    if include and not exclude:
+        return True
+
+
+def get_gz_compressed_file_content(location):
+    """
+    Uncompress a compressed file at `location` and return its content as a byte
+    string. Raise Exceptions on errors.
+    """
+    with gzip.GzipFile(location, "rb") as compressed:
+        content = compressed.read()
+    return content