Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
555 commits
Select commit Hold shift + click to select a range
44a2345
Do not use a trailing comma after "**kwargs" #2233
pombredanne Sep 21, 2020
d279d3a
Extract minimal archive handlimg from test #2233
pombredanne Sep 21, 2020
b58f030
Add support for minimal Gzip decompression #2233
pombredanne Sep 21, 2020
6f18e1a
Provide more details for Win CI heisenbug #2233
pombredanne Sep 21, 2020
609f34e
Fix Click progressbar cli test
steven-esser Sep 23, 2020
6ce679e
Allow symlinks in is_file, is_dir, and walk
JonoYang Oct 19, 2020
9ab840a
Test is_file, is_dir, and walk on symlinks
JonoYang Oct 19, 2020
01f3b71
Update docstring for walk
JonoYang Oct 20, 2020
ced7a9e
Replace allow_symlinks with follow_symlinks
JonoYang Oct 20, 2020
40faf92
Replace allow_symlinks with follow_symlinks
JonoYang Oct 20, 2020
ab07e11
Merge pull request #2 from nexB/visit-symlinks
JonoYang Oct 20, 2020
79f9042
Merge pull request #2 from nexB/visit-symlinks
JonoYang Oct 20, 2020
73dc0ca
Remove encoding keyword argument
tdruez Nov 19, 2020
276a4ad
Remove the 2nd occurence of the encoding keyword argument
tdruez Nov 19, 2020
e8f7afa
Add `src` folder to pass CI tests and RTD builds
AyanSinhaMahapatra Dec 16, 2020
e45322c
Remove Python 2 support
pombredanne Jan 14, 2021
d3eef71
Remove Python 2 support
pombredanne Jan 14, 2021
5617fc2
Merge latest main
pombredanne Jan 14, 2021
6cd894f
Merge latest main
pombredanne Jan 14, 2021
24a04c4
Merge branch 6-encoding-keyword in main
pombredanne Jan 14, 2021
6bcc0e0
Update license and boilerplate files
pombredanne Jan 14, 2021
60bcfb5
Update license and boilerplate files
pombredanne Jan 14, 2021
707fb54
Skip synlink test on Windows
pombredanne Jan 14, 2021
0c1d892
Attempt to make test work on Azure and Windows
pombredanne Jan 14, 2021
da77220
Attempt to make test work on Azure and Windows
pombredanne Jan 14, 2021
23ac916
Format code
pombredanne Jan 14, 2021
bf58b55
Improve documentation
pombredanne Jan 14, 2021
079324c
Improve documentation
pombredanne Jan 14, 2021
ec13282
Merge latest skeleton update-boilerplate branch
pombredanne Jan 14, 2021
219df2c
Merge latest skeleton update-boilerplate branch
pombredanne Jan 14, 2021
fe40dd2
Merge pull request #19 from nexB/update-boilerplate
steven-esser Jan 19, 2021
f088ba6
Move tests one dir level up
pombredanne Jan 21, 2021
b8e2ee9
Improve error reporting if DLL path is missing
pombredanne Jan 21, 2021
fdbc325
Merge pull request #9 from nexB/v21.01.21-prep
pombredanne Jan 22, 2021
3417bdc
Merge pull request #9 from nexB/v21.01.21-prep
pombredanne Jan 22, 2021
627cf0d
Fixed Full Root scans for Virtual Codebase
pratik0316 Apr 4, 2021
cb24545
Fixed Full Root scans for Virtual Codebase
pratik0316 Apr 4, 2021
3d5dfc6
Remove unused import
pombredanne Apr 28, 2021
aa06902
Add Resource.get_path(strip_root) function
pombredanne Apr 28, 2021
691ef8e
Add Resource.get_path(strip_root) function
pombredanne Apr 28, 2021
9c9b87b
Merge branch main from https://github.com/Pratikrocks/commoncode
pombredanne Apr 28, 2021
8cb205e
Merge branch main from https://github.com/Pratikrocks/commoncode
pombredanne Apr 28, 2021
2884d2f
Merge remote-tracking branch 'skeleton/main' into v21.4.28-prep
pombredanne Apr 28, 2021
9f3ef8d
Merge pull request #16 from nexB/v21.4.28-prep
pombredanne Apr 28, 2021
970c7f5
Merge pull request #16 from nexB/v21.4.28-prep
pombredanne Apr 28, 2021
1ecf7de
Add Python 3.10
pombredanne Apr 30, 2021
c2148f7
Remove empty skeleton READMEs
pombredanne May 5, 2021
76ce5e1
Remove empty skeleton READMEs
pombredanne May 5, 2021
1ce1d9f
Improve running command in subprocesses.
pombredanne May 5, 2021
6d47362
Improve running command in subprocesses.
pombredanne May 5, 2021
94b9c09
Mae tests pass on macOS
pombredanne May 5, 2021
a0dbb66
Add common of use of shell=True on Windows
pombredanne May 11, 2021
cda668b
Add must_exist argument to get_test_loc()
pombredanne May 12, 2021
22195c4
Adopt streamlined notices from ScanCode
pombredanne May 12, 2021
1e67f4c
Adopt streamlined notices from ScanCode
pombredanne May 12, 2021
1e35bf1
Update expected resulsts for tests
pombredanne May 12, 2021
3c57dcc
Reorder tests assertions as result == expected
pombredanne May 12, 2021
883e0c4
Reorder tests assertions as result == expected
pombredanne May 12, 2021
d7b73c0
Fix CI tests on Windows
pombredanne May 12, 2021
868c216
Merge pull request #17 from nexB/v21.5.5-prep
pombredanne May 12, 2021
055a111
Merge pull request #17 from nexB/v21.5.5-prep
pombredanne May 12, 2021
4d37645
Remove un-needed click unicode workaround
pombredanne May 25, 2021
c2c539e
Format test code for clarity
pombredanne May 25, 2021
5c506a5
Use named arguments to support future versions
pombredanne May 25, 2021
fcdea9d
Restore handling of click.Option.hidden=True
pombredanne May 25, 2021
ffbc149
Merge pull request #19 from nexB/scancode-2529-click-8
pombredanne May 25, 2021
0f82bca
Merge pull request #19 from nexB/scancode-2529-click-8
pombredanne May 25, 2021
dcbd143
Do not fail with null sizes and counts
pombredanne Jun 10, 2021
a064d21
Do not fail with null sizes and counts
pombredanne Jun 10, 2021
447f083
Merge pull request #22 from nexB/resource-bug
pombredanne Jun 10, 2021
59cde3c
Merge pull request #22 from nexB/resource-bug
pombredanne Jun 10, 2021
d3b481c
Add failing test for #23
pombredanne Jun 11, 2021
abae278
Set a root path if one is not present #23
JonoYang Jun 11, 2021
2b61a47
fixed TypeError due to different attributes over directories and files
pratik0316 Jun 10, 2021
31980c0
fixed TypeError due to different attributes over directories and files
pratik0316 Jun 10, 2021
6c37b96
Merge pull request #21 from Pratikrocks/dynamic_attrs
pombredanne Jun 11, 2021
45a7969
Merge pull request #21 from Pratikrocks/dynamic_attrs
pombredanne Jun 11, 2021
293f0cb
Merge remote-tracking branch 'origin/main' into 23-recursion-error
pombredanne Jun 11, 2021
722deb5
Merge remote-tracking branch 'origin/main' into 23-recursion-error
pombredanne Jun 11, 2021
c6ba586
Do not ignore by default sccs and rcs dirs
pombredanne Jun 11, 2021
45782eb
Do not ignore by default sccs and rcs dirs
pombredanne Jun 11, 2021
9c03034
Merge pull request #24 from nexB/23-recursion-error
pombredanne Jun 11, 2021
6ad7e87
Merge pull request #24 from nexB/23-recursion-error
pombredanne Jun 11, 2021
3adcdae
Add preserve_spaces argument to portable_filename
JonoYang Jul 22, 2021
1fa2d85
Add preserve_spaces argument to portable_filename
JonoYang Jul 22, 2021
dc25e52
Merge pull request #26 from nexB/preserve_spaces_in_filenames
pombredanne Jul 23, 2021
d586269
Merge pull request #26 from nexB/preserve_spaces_in_filenames
pombredanne Jul 23, 2021
51d707d
Display counters in progressbar
pombredanne Aug 27, 2021
ca09bcb
Display counters in progressbar
pombredanne Aug 27, 2021
aa72ac7
Merge pull request #27 from nexB/click-progress
pombredanne Aug 27, 2021
1c652bc
Merge pull request #27 from nexB/click-progress
pombredanne Aug 27, 2021
b56a9db
Add output data format version to header
AyanSinhaMahapatra Aug 30, 2021
29c294e
Merge pull request #28 from AyanSinhaMahapatra/2653-add-output-format…
pombredanne Aug 30, 2021
1bb5556
Also support click 7.1.+ #31
pombredanne Sep 24, 2021
6d745f4
Also support click 7.1.+ #31
pombredanne Sep 24, 2021
7c8f7e2
Merge pull request #32 from nexB/23-support-many-click-versions
pombredanne Sep 24, 2021
0dafa91
Merge pull request #32 from nexB/23-support-many-click-versions
pombredanne Sep 24, 2021
99b2b72
Add black codestyle test for skeleton
keshav-space Feb 22, 2022
090e8d1
Add warnings field to codebase headers
JonoYang Mar 28, 2022
3efea78
Merge remote-tracking branch 'skeleton/main' into 38-add-warnings-to-…
JonoYang Mar 29, 2022
865e124
Merge pull request #39 from nexB/38-add-warnings-to-codebase-headers
JonoYang Apr 5, 2022
6c0c35a
Merge pull request #39 from nexB/38-add-warnings-to-codebase-headers
JonoYang Apr 5, 2022
9f45ea8
Add functions to extract name/version of filename
pombredanne Nov 24, 2021
7bb2698
Merge pull request #33 from nexB/name-version-from-file
JonoYang Apr 5, 2022
3262714
Use shutil.get_terminal_size()
georgthegreat Apr 17, 2022
45319c8
Merge pull request #40 from georgthegreat/patch-1
AyanSinhaMahapatra Apr 18, 2022
7c2100f
Enable cache to work with a single resource
pombredanne Apr 29, 2022
9af719d
Merge remote-tracking branch 'uni/main' into main
pombredanne Apr 29, 2022
5f37e25
Merge remote-tracking branch 'uni/main' into main
pombredanne Apr 29, 2022
8a5d3cf
Merge remote-tracking branch 'skeleton/main' into improve-resources
pombredanne Apr 29, 2022
43038cc
Merge remote-tracking branch 'skeleton/main' into improve-resources
pombredanne Apr 29, 2022
c1dd913
Merge pull request #41 from nexB/improve-resources
pombredanne Apr 29, 2022
d013e06
Merge pull request #41 from nexB/improve-resources
pombredanne Apr 29, 2022
0aa6484
Use paths in Codebase
pombredanne May 13, 2022
b514e5f
Use paths in Codebase
pombredanne May 13, 2022
425b0c0
Fix safe_path test
pombredanne May 13, 2022
6f0d01c
Re-enable tests that were disabled
pombredanne May 13, 2022
9390126
Test combining scans with shared directories
JonoYang May 14, 2022
5c62534
Improve how we collect Resource.children
pombredanne May 14, 2022
ac0cacf
Revert changes on combining scan data
pombredanne May 14, 2022
d8d8022
Add new virtualcocdebase test
pombredanne May 14, 2022
033d0c6
Remove "u" unicode string prefixes.
pombredanne May 15, 2022
1387cc2
Format code
pombredanne May 15, 2022
ffa254e
Update resource and codebase processing
pombredanne May 16, 2022
501ceda
Update resource and codebase processing
pombredanne May 16, 2022
3ddd50c
Enable a paths argument in Codebase/VirtualCodebase
pombredanne May 16, 2022
532b6f4
Enable a paths argument in Codebase/VirtualCodebase
pombredanne May 16, 2022
e9fe881
Remove debug trace output
pombredanne May 16, 2022
f1fb843
Add a new Resource.parent_path() function
pombredanne May 16, 2022
e1a88a9
Format JSON
pombredanne May 17, 2022
81cbddc
Drop strip_root/full_root codebase attributes
pombredanne May 17, 2022
5458954
Drop strip_root/full_root codebase attributes
pombredanne May 17, 2022
0269ba4
Make Resource.build_path work on Windows
pombredanne May 17, 2022
0ae33b8
Update docstrings
JonoYang Jul 29, 2022
d3d869b
Merge pull request #42 from nexB/use-paths-in-codebase
JonoYang Jul 29, 2022
96a8748
Merge pull request #42 from nexB/use-paths-in-codebase
JonoYang Jul 29, 2022
4351cf7
Create POSIX safe filename function
JonoYang Jun 11, 2022
7c318ca
Update portable_filename with new posix option
JonoYang Jun 11, 2022
6140c8d
Update safe_path function signature
JonoYang Jun 11, 2022
2b6401e
Test posix_only argument for portable_filename
JonoYang Aug 2, 2022
8d55db3
Test posix_only argument for portable_filename
JonoYang Aug 2, 2022
dc4c704
Add missing docstring
pombredanne Jul 29, 2022
e136d93
Merge pull request #44 from nexB/posix-safe-filename
pombredanne Aug 24, 2022
9fb03ad
Merge pull request #44 from nexB/posix-safe-filename
pombredanne Aug 24, 2022
c4a2f6f
Merge branch 'no-intbitset' into main
pombredanne Aug 24, 2022
f72d7a4
Merge pull request #45 from nexB/release31-prep
pombredanne Aug 24, 2022
bc44e3d
Do not assign None to codebase attributes #48
JonoYang Dec 1, 2022
eb4940b
Do not assign None to codebase attributes #48
JonoYang Dec 1, 2022
adad466
Merge pull request #49 from nexB/48-correctly-assign-codebase-attributes
JonoYang Feb 23, 2023
cca0759
Merge pull request #49 from nexB/48-correctly-assign-codebase-attributes
JonoYang Feb 23, 2023
83f8d5a
Remove unused test files
pombredanne May 19, 2023
24d21bb
Update test file to non-proprietary license variant
pombredanne May 19, 2023
def3c30
Remove unused test files
pombredanne May 19, 2023
8a89d6c
Merge pull request #53 from nexB/remove-unused-test-files
pombredanne May 19, 2023
eb20bae
Properly append root path segment #57
JonoYang Aug 24, 2023
7b52567
Properly append root path segment #57
JonoYang Aug 24, 2023
07c335d
Update _get_parent_directory #57
JonoYang Aug 25, 2023
b557258
Update _get_parent_directory #57
JonoYang Aug 25, 2023
a111c0e
Merge pull request #58 from nexB/fix-virtualcodebase-creation-bug
JonoYang Aug 25, 2023
982e532
Merge pull request #58 from nexB/fix-virtualcodebase-creation-bug
JonoYang Aug 25, 2023
f2dfa42
Add support for FIPS
pombredanne Dec 28, 2022
a505485
Merge pull request #54 from nexB/fips
JonoYang Aug 25, 2023
bace0e3
Replace the deprecated datetime.datetime.utcfromtimestamp
stefan6419846 Jan 13, 2024
45cb140
Add on_macos_arm64 and on_ubuntu_22 to commoncode.system
JonoYang May 15, 2024
15cfbf1
Prep for new release
JonoYang May 15, 2024
bab3a84
Merge pull request #63 from nexB/add-new-system-markers
JonoYang May 15, 2024
97cfa5e
Move parse_os_release from container-inspector to commoncode
JonoYang May 16, 2024
99bda71
Move parse_os_release from container-inspector to commoncode
JonoYang May 16, 2024
6f98558
Merge pull request #64 from nexB/os-release-parser
JonoYang May 16, 2024
b949eca
Merge pull request #64 from nexB/os-release-parser
JonoYang May 16, 2024
0601fa6
Use parse_os_release in commoncode.system
JonoYang May 16, 2024
f5bbb6a
Merge pull request #65 from nexB/os-release-parser
JonoYang May 16, 2024
1f05e80
don't fail if there is no /etc/os-release
dotlambda Jul 24, 2024
e76bd82
#67 - Update link references of ownership from nexB to aboutcode-org
chinyeungli Aug 14, 2024
0ee1b11
#67 - Update link references of ownership from nexB to aboutcode-org
chinyeungli Aug 14, 2024
b9474b9
Update link references of ownership from nexB to aboutcode-org
chinyeungli Aug 20, 2024
c31cc31
Add new arg to gen missing test files
pombredanne Sep 4, 2024
7c78724
Merge branch 'main' into patch-1
pombredanne Sep 5, 2024
a54e1b2
Merge pull request #62 from stefan6419846/utcfromtimestamp
pombredanne Sep 5, 2024
f88dd20
Merge pull request #66 from dotlambda/patch-1
pombredanne Sep 5, 2024
1e5ef30
Merge remote-tracking branch 'origin/main' into pre-release
pombredanne Sep 5, 2024
5725110
Switch to chardet for stable behavior
pombredanne Sep 5, 2024
69819f1
Adopt black and isort style
pombredanne Sep 5, 2024
07e1b29
Adopt black and isort style
pombredanne Sep 5, 2024
3e8a88a
Merge pull request #68 from aboutcode-org/prep-release
pombredanne Sep 5, 2024
088d69a
Merge pull request #68 from aboutcode-org/prep-release
pombredanne Sep 5, 2024
a07aa55
Compute checksums from binary stream
pombredanne Dec 6, 2024
69295c5
Compute checksums from binary stream
pombredanne Dec 6, 2024
0e9f1e4
Apply formatting
pombredanne Dec 6, 2024
fda216b
Apply formatting
pombredanne Dec 6, 2024
35ec16c
Update docstirngs and python version
AyanSinhaMahapatra Dec 6, 2024
5945a39
Merge remote-tracking branch 'skeleton/main' into 72-large-file-hash
AyanSinhaMahapatra Dec 6, 2024
6d756d1
Add style checks to CI
AyanSinhaMahapatra Dec 6, 2024
60b7ca7
Merge pull request #73 from aboutcode-org/72-large-file-hash
AyanSinhaMahapatra Dec 6, 2024
5619367
Merge pull request #73 from aboutcode-org/72-large-file-hash
AyanSinhaMahapatra Dec 6, 2024
035b03f
migrate os.listdir() to os.scandir() to increase performance
xsuchy Jan 17, 2025
b59974d
Adjust `as_unicode` for breaking bs4 4.13 changes (#80)
jloehel Feb 14, 2025
906214a
Fix empty file checksum
AyanSinhaMahapatra Mar 6, 2025
beeaa12
Fix empty file checksum
AyanSinhaMahapatra Mar 6, 2025
67a054d
Merge branch 'main' into scandir
AyanSinhaMahapatra Mar 6, 2025
389f036
Migrate remaining os.listdir to os.scandir
AyanSinhaMahapatra Mar 6, 2025
ce95b68
Migrate remaining os.listdir to os.scandir
AyanSinhaMahapatra Mar 6, 2025
8c32378
Merge pull request #82 from aboutcode-org/fix-empty-file-checksum
AyanSinhaMahapatra Mar 6, 2025
27c1a44
Merge pull request #82 from aboutcode-org/fix-empty-file-checksum
AyanSinhaMahapatra Mar 6, 2025
54074b7
Merge pull request #83 from aboutcode-org/scandir
AyanSinhaMahapatra Mar 6, 2025
7bf3a66
Merge pull request #83 from aboutcode-org/scandir
AyanSinhaMahapatra Mar 6, 2025
0928e69
Reformat test code
pombredanne Mar 29, 2025
0de6178
Format code
pombredanne Mar 29, 2025
14c5a7f
Reformat test code
pombredanne Mar 29, 2025
9477f63
Format code
pombredanne Mar 29, 2025
1acbb70
Update codestyle command
JonoYang Mar 31, 2025
52256f2
Merge branch 'add-code-style-checks' of https://github.com/nexB/skeleton
pombredanne Apr 1, 2025
af9b2a8
Add code checks to CI
pombredanne Apr 1, 2025
78a774e
Merge pull request #96 from aboutcode-org/add-code-style-checks
pombredanne Apr 4, 2025
47c985d
Fix using incorrect hidden property on ProgressBar instance
piiq May 19, 2025
121ae02
Fix compatibility issues from click >= 8.2.0
AyanSinhaMahapatra Jun 11, 2025
62bee4e
Merge pull request #89 from aboutcode-org/bugfix/progressbar-hidden-p…
AyanSinhaMahapatra Jun 11, 2025
a92cdc4
Merge remote-tracking branch 'skeleton/main' into release-prep-32.3.0
AyanSinhaMahapatra Jun 11, 2025
a0d4471
Fix style check failures
AyanSinhaMahapatra Jun 11, 2025
8d6731b
Fix style check failures
AyanSinhaMahapatra Jun 11, 2025
6c5e60f
Merge pull request #90 from aboutcode-org/release-prep-32.3.0
AyanSinhaMahapatra Jun 11, 2025
c6a4a8b
Merge pull request #90 from aboutcode-org/release-prep-32.3.0
AyanSinhaMahapatra Jun 11, 2025
f2ede7a
fix: handle paths with non-utf-8 bytes
abraemer Oct 17, 2025
bc77449
fix: handle paths with non-utf-8 bytes
abraemer Oct 17, 2025
8f141fd
Do not set flag_value as None in click Option
AyanSinhaMahapatra Oct 21, 2025
97ab386
Drop python3.9 support and add python3.14
AyanSinhaMahapatra Oct 21, 2025
f351f90
Replace is_hidden with hidden in ProgressBar
AyanSinhaMahapatra Oct 21, 2025
0c03a50
Remove special file to fix py3.14 tests
AyanSinhaMahapatra Oct 22, 2025
af61a83
Revert to CompatProgressBar to support old click versions
AyanSinhaMahapatra Oct 22, 2025
b96d2eb
Fix string quotes in test for non-UTF8 path
AyanSinhaMahapatra Oct 22, 2025
8b885ed
Fix formatting in test_resource.py
AyanSinhaMahapatra Oct 22, 2025
753feb9
Merge pull request #91 from abraemer/fix-path-encoding-for-hash
AyanSinhaMahapatra Oct 22, 2025
1730ba8
Merge pull request #91 from abraemer/fix-path-encoding-for-hash
AyanSinhaMahapatra Oct 22, 2025
2801bc8
Merge pull request #92 from aboutcode-org/fix-click-compat
JonoYang Oct 22, 2025
e8e4de0
Merge pull request #92 from aboutcode-org/fix-click-compat
JonoYang Oct 22, 2025
f10c7d6
Update system.py for py314
AyanSinhaMahapatra Jan 7, 2026
02f65e3
Update system.py for py314
AyanSinhaMahapatra Jan 7, 2026
d7aa394
Merge pull request #94 from aboutcode-org/update-system-py
AyanSinhaMahapatra Jan 7, 2026
e64968a
Merge pull request #94 from aboutcode-org/update-system-py
AyanSinhaMahapatra Jan 7, 2026
1b80d7c
Support setting the tarfile extract filter
AyanSinhaMahapatra Jan 8, 2026
1b3c5c3
Support setting the tarfile extract filter
AyanSinhaMahapatra Jan 8, 2026
8169888
Merge pull request #95 from aboutcode-org/update-tar-extract
AyanSinhaMahapatra Jan 8, 2026
08b98dc
Merge pull request #95 from aboutcode-org/update-tar-extract
AyanSinhaMahapatra Jan 8, 2026
4cff6f9
Add sunos and haiku
JonoYang Apr 8, 2026
7470049
Merge pull request #99 from aboutcode-org/add-os
JonoYang Apr 13, 2026
b0146ee
Add openbsd to os_arch()
JonoYang Apr 28, 2026
288cc8a
Merge pull request #103 from aboutcode-org/add-openbsd
JonoYang May 1, 2026
8404334
Merge branch 'main' of ../commoncode-filter into fix-ignore-pre-scan-…
AyanSinhaMahapatra May 22, 2026
cce6b4f
Merge branch 'main' of ../commoncode-filter into fix-ignore-pre-scan-…
AyanSinhaMahapatra May 22, 2026
da248c8
Remove commoncode as a dependency
AyanSinhaMahapatra May 22, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
1 change: 0 additions & 1 deletion pyproject-scancode-toolkit-mini.toml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ dependencies = [
"click >= 6.7, !=7.0, !=8.1.8;python_version<'3.10'",
"click >= 8.2.0;python_version>='3.10'",
"colorama >= 0.3.9",
"commoncode >= 32.4.2",
"cyseq >= 0.1.2",
"container-inspector >= 31.0.0",
"debian-inspector >= 31.1.0",
Expand Down
1 change: 0 additions & 1 deletion pyproject-scancode-toolkit.toml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ dependencies = [
"click >= 6.7, !=7.0, !=8.1.8;python_version<'3.10'",
"click >= 8.2.0;python_version>='3.10'",
"colorama >= 0.3.9",
"commoncode >= 32.4.2",
"cyseq >= 0.1.2",
"container-inspector >= 31.0.0",
"debian-inspector >= 31.1.0",
Expand Down
1 change: 0 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ dependencies = [
"click >= 6.7, !=7.0, !=8.1.8;python_version<'3.10'",
"click >= 8.2.0;python_version>='3.10'",
"colorama >= 0.3.9",
"commoncode >= 32.4.2",
"cyseq >= 0.1.2",
"container-inspector >= 31.0.0",
"debian-inspector >= 31.1.0",
Expand Down
1 change: 0 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ chardet==5.2.0
charset-normalizer==3.4.4
click==8.3.1
colorama==0.4.6
commoncode==32.4.2
construct==2.10.70
container-inspector==33.0.0
cryptography==46.0.5
Expand Down
29 changes: 29 additions & 0 deletions src/commoncode/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#
# Copyright (c) nexB Inc. and others. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
# See https://github.com/aboutcode-org/commoncode for support or download.
# See https://aboutcode.org for more information about nexB OSS projects.
#


def set_re_max_cache(max_cache=1000000):
"""
Set re and fnmatch _MAXCACHE to 1Million items to cache compiled regex
aggressively. Their default is a maximum of 100 items and many utilities and
libraries use a lot of regexes: therefore 100 is not enough to benefit from
caching.
"""
import fnmatch
import re

remax = getattr(re, "_MAXCACHE", 0)
if remax < max_cache:
setattr(re, "_MAXCACHE", max_cache)

fnmatchmax = getattr(fnmatch, "_MAXCACHE", 0)
if fnmatchmax < max_cache:
setattr(fnmatch, "_MAXCACHE", max_cache)


set_re_max_cache()
133 changes: 133 additions & 0 deletions src/commoncode/archive.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
#
# Copyright (c) nexB Inc. and others. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
# See https://github.com/aboutcode-org/commoncode for support or download.
# See https://aboutcode.org for more information about nexB OSS projects.
#

import gzip
import os
import tarfile
import zipfile
from functools import partial
from os import path

from commoncode.system import on_windows
from commoncode.system import py314

"""
Mimimal tar and zip file handling, primarily for testing.
"""


def _extract_tar_raw(test_path, target_dir, to_bytes, *args, **kwargs):
"""
Raw simplified extract for certain really weird paths and file
names.
"""
tar = None
try:
tar = tarfile.open(test_path)
tar.extractall(path=target_dir)
finally:
if tar:
tar.close()


extract_tar_raw = partial(_extract_tar_raw, to_bytes=True)

extract_tar_uni = partial(_extract_tar_raw, to_bytes=False)


def extract_tar(location, target_dir, verbatim=False, filter=None, *args, **kwargs):
"""
Extract a tar archive at location in the target_dir directory.
If `verbatim` is True preserve the permissions.
"""
# always for using bytes for paths on all OSses... tar seems to use bytes internally
# and get confused otherwise
location = os.fsencode(location)
with open(location, "rb") as input_tar:
tar = None
try:
tar = tarfile.open(fileobj=input_tar)
tarinfos = tar.getmembers()
to_extract = []
for tarinfo in tarinfos:
if tar_can_extract(tarinfo, verbatim):
if not verbatim:
tarinfo.mode = 0o755
to_extract.append(tarinfo)
if py314 and filter:
tar.extractall(target_dir, members=to_extract, filter=filter)
else:
tar.extractall(target_dir, members=to_extract)
finally:
if tar:
tar.close()


def extract_zip(location, target_dir, *args, **kwargs):
"""
Extract a zip archive file at location in the target_dir directory.
"""
if not path.isfile(location) and zipfile.is_zipfile(location):
raise Exception("Incorrect zip file %(location)r" % locals())

with zipfile.ZipFile(location) as zipf:
for info in zipf.infolist():
name = info.filename
content = zipf.read(name)
target = path.join(target_dir, name)
if not path.exists(path.dirname(target)):
os.makedirs(path.dirname(target))
if not content and target.endswith(path.sep):
if not path.exists(target):
os.makedirs(target)
if not path.exists(target):
with open(target, "wb") as f:
f.write(content)


def extract_zip_raw(location, target_dir, *args, **kwargs):
"""
Extract a zip archive file at location in the target_dir directory.
Use the builtin extractall function
"""
if not path.isfile(location) and zipfile.is_zipfile(location):
raise Exception("Incorrect zip file %(location)r" % locals())

with zipfile.ZipFile(location) as zipf:
zipf.extractall(path=target_dir)


def tar_can_extract(tarinfo, verbatim):
"""
Return True if a tar member can be extracted to handle OS specifics.
If verbatim is True, always return True.
"""
if tarinfo.ischr():
# never extract char devices
return False

if verbatim:
# extract all on all OSse
return True

# FIXME: not sure hard links are working OK on Windows
include = tarinfo.type in tarfile.SUPPORTED_TYPES
exclude = tarinfo.isdev() or (on_windows and tarinfo.issym())

if include and not exclude:
return True


def get_gz_compressed_file_content(location):
"""
Uncompress a compressed file at `location` and return its content as a byte
string. Raise Exceptions on errors.
"""
with gzip.GzipFile(location, "rb") as compressed:
content = compressed.read()
return content
Loading
Loading