[pull] master from kevoreilly:master

conf/default/web.conf.default

@@ -66,8 +66,8 @@ expose_process_log = no
 reprocess_tasks = no
 # Allow to reprocess failed processing tasks
 reprocess_failed_processing = no
-# Allows you to define URL splitter, "," is default
-url_splitter = ,
+# Allows you to define URL splitter (for example comma). Empty value disables splitting
+url_splitter =
 # Limit number of files extracted from archive in demux.py
 demux_files_limit = 10
 public_searches = yes

installer/kvm-qemu.sh

@@ -135,12 +135,19 @@ BXPC_REPLACER='<WOOT>'
 # what to use as a replacement for seabios in config.h (e.g., 'AMI', 'Award', 'Phoenix')
 BOCHS_SEABIOS_BLOCK_REPLACER='<WOOT>'
 
+# If apt-daily-upgrade.timer should be masked or not.
+# 1 = mask it
+# 0 = don't mask it
+MASK_APT_DAILY_UPGRADE=1
 
 # if a config file is present, read it in
 if [ -f "./kvm-config.sh" ]; then
         . ./kvm-config.sh
 fi
 
+if [ "$MASK_APT_DAILY_UPGRADE" -ge 1 ]; then
+	systemctl mask apt-daily-upgrade.timer
+fi
 
 # ToDO add to see if cpu supports VTx
 # egrep '(vmx|svm)' --color=always /proc/cpuinfo

lib/cuckoo/core/data/tasking.py

@@ -201,6 +201,9 @@ def add(
         else:
             return None
 
+        if tlp:
+            tlp = tlp.lower()
+
         task.category = obj.__class__.__name__.lower()
         task.timeout = timeout
         task.package = package

lib/cuckoo/core/startup.py

@@ -121,26 +121,45 @@ def check_webgui_mongo():
         # Create an index based on the info.id dict key. Increases overall scalability
         # with large amounts of data.
         # Note: Silently ignores the creation if the index already exists.
-        mongo_create_index("analysis", "info.id", name="info.id_1")
-        mongo_create_index("files", [("_task_ids", 1)])
-
+        index_configs = [
+            ("analysis", [("info.id", -1)], {"name": "info_id_desc"}),
+            ("files", [("_task_ids", 1)], {}),
+        ]
         if repconf.mongodb.get("index_yara", False):
-            mongo_create_index("files", "yara.name", name="yara_name")
-            mongo_create_index("files", "cape_yara.name", name="cape_yara_name")
-
+            index_configs.extend([
+                ("files", "yara.name", {"name": "yara_name"}),
+                ("files", "cape_yara.name", {"name": "cape_yara_name"}),
+            ])
         if repconf.mongodb.get("index_clamav", False):
-            mongo_create_index("files", "clamav", name="clamav_index")
-
+            index_configs.append(("files", "clamav", {"name": "clamav_index"}))
         if repconf.mongodb.get("index_hashes", False):
-            mongo_create_index("files", "md5", name="file_md5")
-            mongo_create_index("files", "sha1", name="file_sha1")
-            mongo_create_index("files", "ssdeep", name="file_ssdeep")
-
+            index_configs.extend([
+                ("files", "md5", {"name": "file_md5"}),
+                ("files", "sha1", {"name": "file_sha1"}),
+                ("files", "ssdeep", {"name": "file_ssdeep"}),
+            ])
         if repconf.mongodb.get("index_detections", False):
-            mongo_create_index("analysis", "detections.family", name="detections_family")
-
+            index_configs.append(("analysis", [("detections.family", 1), ("_id", -1)], {"name": "detections_family_id_desc"}))
         if repconf.mongodb.get("index_filenames", False):
-            mongo_create_index("analysis", "target.file.name", name="name_1")
+            index_configs.append(("analysis", [("target.file.name", 1), ("_id", -1)], {"name": "target_file_name_id_desc"}))
+
+        # Obsolete indexes to drop
+        obsolete_indexes = {
+            "analysis": ["info.id_1", "detections_family", "name_1", "detections_family_1", "target_file_name_1"],
+        }
+
+        for coll, keys, kwargs in index_configs:
+            try:
+                mongo_create_index(coll, keys, **kwargs)
+            except Exception as e:
+                log.warning("Failed to create MongoDB index %s on %s: %s", kwargs.get("name", keys), coll, e)
+
+        # Drop obsolete indexes
+        from dev_utils.mongodb import results_db
+        for coll, indexes in obsolete_indexes.items():
+            for index_name in indexes:
+                with suppress(Exception):
+                    getattr(results_db, coll).drop_index(index_name)
     elif repconf.elasticsearchdb.enabled:
         # ToDo add check
         pass

utils/go-fetcher/go.mod

@@ -1,6 +1,6 @@
 module go-fetcher
 
-go 1.24.0
+go 1.25.0
 
 require (
 	gorm.io/driver/mysql v1.5.2
@@ -12,12 +12,12 @@ require (
 require (
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.5.4 // indirect
-	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/pgx/v5 v5.9.2 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/mattn/go-sqlite3 v1.14.17 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
 	golang.org/x/text v0.31.0 // indirect
 )

utils/go-fetcher/go.sum

@@ -5,12 +5,12 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
-github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
-github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8=
-github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
-github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
-github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
+github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -22,10 +22,10 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
 golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=