From f9eb20c5e6142b4133d81716f38edef81acceca6 Mon Sep 17 00:00:00 2001 From: peterolkhov Date: Mon, 9 Nov 2026 17:25:43 -0500 Subject: [PATCH 1/4] fix: preserve JSON scalar input types in local storage --- .../_file_system/_key_value_store_client.py | 5 +- .../unit/storage_clients/test_file_system.py | 87 +++++++++++++++++-- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/apify/storage_clients/_file_system/_key_value_store_client.py b/src/apify/storage_clients/_file_system/_key_value_store_client.py index f8cd47a03..822e61416 100644 --- a/src/apify/storage_clients/_file_system/_key_value_store_client.py +++ b/src/apify/storage_clients/_file_system/_key_value_store_client.py @@ -127,6 +127,7 @@ async def _create_missing_metadata_for_input_file(key: str, record_path: Path) - # Figure out the metadata from the file content size = len(content) + is_json = record_path.suffix == '.json' if record_path.suffix == '.json': value = json.loads(content.decode('utf-8')) elif record_path.suffix == '.txt': @@ -136,10 +137,12 @@ async def _create_missing_metadata_for_input_file(key: str, record_path: Path) - value = json.loads(content.decode('utf-8')) except json.JSONDecodeError: value = content + else: + is_json = True else: value = content - content_type = infer_mime_type(value) + content_type = 'application/json; charset=utf-8' if is_json else infer_mime_type(value) record_metadata = KeyValueStoreRecordMetadata(key=key, content_type=content_type, size=size) record_metadata_filepath = record_path.with_name(f'{record_path.name}.{METADATA_FILENAME}') diff --git a/tests/unit/storage_clients/test_file_system.py b/tests/unit/storage_clients/test_file_system.py index 6b30dfcb5..8e98cca2a 100644 --- a/tests/unit/storage_clients/test_file_system.py +++ b/tests/unit/storage_clients/test_file_system.py @@ -3,6 +3,7 @@ import asyncio import json from pathlib import Path +from typing import Any import pytest @@ -64,8 +65,30 @@ async def test_purge_preserves_input_file_and_metadata() -> None: assert json.loads(input_content) == json.loads('{"test": "input"}') -@pytest.mark.parametrize('input_file_name', ['INPUT', 'INPUT.json']) -async def test_pre_existing_input_used_by_actor(input_file_name: str) -> None: +@pytest.mark.parametrize( + 'input_file_name', + [pytest.param('INPUT', id='bare'), pytest.param('INPUT.json', id='json')], +) +@pytest.mark.parametrize( + 'pre_existing_input', + [ + pytest.param({'foo': 'bar'}, id='object'), + pytest.param({}, id='empty-object'), + pytest.param([1, 2], id='array'), + pytest.param([], id='empty-array'), + pytest.param('demo', id='string'), + pytest.param('', id='empty-string'), + pytest.param(42, id='integer'), + pytest.param(0, id='zero-integer'), + pytest.param(1.5, id='float'), + pytest.param(0.0, id='zero-float'), + pytest.param(True, id='true'), + pytest.param(False, id='false'), + pytest.param(None, id='null'), + ], +) +async def test_pre_existing_input_used_by_actor(input_file_name: str, pre_existing_input: Any) -> None: + """Pre-existing JSON input preserves its value, type, bytes, and serialization metadata.""" configuration = Configuration() service_locator.set_configuration(configuration) @@ -74,15 +97,24 @@ async def test_pre_existing_input_used_by_actor(input_file_name: str) -> None: path_to_input.mkdir(parents=True) assert list(path_to_input.glob('*')) == [] - pre_existing_input = { - 'foo': 'bar', - } - - # Create pre-existing INPUT.json file - (path_to_input / input_file_name).write_text(json.dumps(pre_existing_input)) + input_file = path_to_input / input_file_name + input_bytes = json.dumps(pre_existing_input).encode('utf-8') + input_file.write_bytes(input_bytes) async with Actor(): - assert pre_existing_input == await Actor.get_input() + actual_input = await Actor.get_input() + assert actual_input == pre_existing_input + assert type(actual_input) is type(pre_existing_input) + store = await Actor.open_key_value_store() + stored_input = await store.get_value(configuration.input_key) + assert stored_input == pre_existing_input + assert type(stored_input) is type(pre_existing_input) + + assert input_file.read_bytes() == input_bytes + metadata = json.loads((path_to_input / f'{input_file_name}.{METADATA_FILENAME}').read_text()) + assert metadata['content_type'] == 'application/json; charset=utf-8' + assert metadata['key'] == configuration.input_key + assert metadata['size'] == len(input_bytes) # Make sure that the input file doesn't get renamed in the process and metadata are added assert set(path_to_input.glob('*')) == { @@ -132,3 +164,40 @@ async def test_record_exists_and_delete_value_target_existing_input_file() -> No await client.delete_value(key=configuration.input_key) assert await client.record_exists(key=configuration.input_key) is False assert not (kvs_path / 'INPUT.json').exists() + + +async def test_pre_existing_input_metadata_is_preserved() -> None: + """Opening an input record respects its explicit content type and leaves its metadata unchanged.""" + configuration = Configuration.get_global_configuration() + kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default' + kvs_path.mkdir(parents=True) + input_file = kvs_path / 'INPUT.json' + input_file.write_bytes(b'42') + metadata_file = kvs_path / f'INPUT.json.{METADATA_FILENAME}' + metadata_bytes = json.dumps( + {'key': configuration.input_key, 'content_type': 'text/plain; charset=utf-8', 'size': 2} + ).encode('utf-8') + metadata_file.write_bytes(metadata_bytes) + + client = await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration) + record = await client.get_value(key=configuration.input_key) + + assert record is not None + assert record.value == '42' + assert input_file.read_bytes() == b'42' + assert metadata_file.read_bytes() == metadata_bytes + + +async def test_malformed_json_input_does_not_create_metadata() -> None: + """Malformed JSON input raises without changing the input or creating record metadata.""" + configuration = Configuration.get_global_configuration() + kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default' + kvs_path.mkdir(parents=True) + input_file = kvs_path / 'INPUT.json' + input_file.write_bytes(b'{not json') + + with pytest.raises(json.JSONDecodeError): + await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration) + + assert input_file.read_bytes() == b'{not json' + assert not (kvs_path / f'INPUT.json.{METADATA_FILENAME}').exists() From 38689c9fc9c1f9434c9ce52085df78df74621df0 Mon Sep 17 00:00:00 2001 From: peterolkhov Date: Wed, 11 Nov 2026 13:06:19 -0500 Subject: [PATCH 2/4] Simplify local input metadata and preserve binary input --- .../_file_system/_key_value_store_client.py | 21 ++++++----- .../unit/storage_clients/test_file_system.py | 36 +++++++++++++++---- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/apify/storage_clients/_file_system/_key_value_store_client.py b/src/apify/storage_clients/_file_system/_key_value_store_client.py index 822e61416..27700480c 100644 --- a/src/apify/storage_clients/_file_system/_key_value_store_client.py +++ b/src/apify/storage_clients/_file_system/_key_value_store_client.py @@ -8,7 +8,7 @@ from typing_extensions import override from crawlee._consts import METADATA_FILENAME -from crawlee._utils.file import atomic_write, infer_mime_type, json_dumps +from crawlee._utils.file import atomic_write, json_dumps from crawlee.configuration import Configuration as CrawleeConfiguration from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata @@ -127,22 +127,21 @@ async def _create_missing_metadata_for_input_file(key: str, record_path: Path) - # Figure out the metadata from the file content size = len(content) - is_json = record_path.suffix == '.json' if record_path.suffix == '.json': - value = json.loads(content.decode('utf-8')) + json.loads(content.decode('utf-8')) + content_type = 'application/json; charset=utf-8' elif record_path.suffix == '.txt': - value = content.decode('utf-8') + content.decode('utf-8') + content_type = 'text/plain; charset=utf-8' elif record_path.suffix == '': try: - value = json.loads(content.decode('utf-8')) - except json.JSONDecodeError: - value = content + json.loads(content.decode('utf-8')) + except (json.JSONDecodeError, UnicodeDecodeError): + content_type = 'application/octet-stream' else: - is_json = True + content_type = 'application/json; charset=utf-8' else: - value = content - - content_type = 'application/json; charset=utf-8' if is_json else infer_mime_type(value) + content_type = 'application/octet-stream' record_metadata = KeyValueStoreRecordMetadata(key=key, content_type=content_type, size=size) record_metadata_filepath = record_path.with_name(f'{record_path.name}.{METADATA_FILENAME}') diff --git a/tests/unit/storage_clients/test_file_system.py b/tests/unit/storage_clients/test_file_system.py index 8e98cca2a..0d930811a 100644 --- a/tests/unit/storage_clients/test_file_system.py +++ b/tests/unit/storage_clients/test_file_system.py @@ -73,16 +73,9 @@ async def test_purge_preserves_input_file_and_metadata() -> None: 'pre_existing_input', [ pytest.param({'foo': 'bar'}, id='object'), - pytest.param({}, id='empty-object'), - pytest.param([1, 2], id='array'), - pytest.param([], id='empty-array'), - pytest.param('demo', id='string'), pytest.param('', id='empty-string'), - pytest.param(42, id='integer'), pytest.param(0, id='zero-integer'), pytest.param(1.5, id='float'), - pytest.param(0.0, id='zero-float'), - pytest.param(True, id='true'), pytest.param(False, id='false'), pytest.param(None, id='null'), ], @@ -124,6 +117,35 @@ async def test_pre_existing_input_used_by_actor(input_file_name: str, pre_existi } +@pytest.mark.parametrize( + 'input_bytes', + [pytest.param(b'hello', id='plain'), pytest.param('hello'.encode('utf-16'), id='utf16')], +) +async def test_pre_existing_non_json_input_used_by_actor(input_bytes: bytes) -> None: + """Bare non-JSON input preserves its bytes and uses binary serialization metadata.""" + configuration = Configuration() + service_locator.set_configuration(configuration) + path_to_input = Path(configuration.storage_dir) / 'key_value_stores' / 'default' + path_to_input.mkdir(parents=True) + input_file = path_to_input / 'INPUT' + input_file.write_bytes(input_bytes) + + async with Actor(): + actual_input = await Actor.get_input() + assert actual_input == input_bytes + assert type(actual_input) is bytes + store = await Actor.open_key_value_store() + stored_input = await store.get_value(configuration.input_key) + assert stored_input == input_bytes + assert type(stored_input) is bytes + + assert input_file.read_bytes() == input_bytes + metadata = json.loads((path_to_input / f'INPUT.{METADATA_FILENAME}').read_text()) + assert metadata['content_type'] == 'application/octet-stream' + assert metadata['key'] == configuration.input_key + assert metadata['size'] == len(input_bytes) + + async def test_set_value_with_input_key_targets_existing_input_file() -> None: """`set_value` with the input key overwrites the existing `INPUT.json` instead of creating a duplicate.""" configuration = Configuration.get_global_configuration() From 5be5190a5d3d3ec424beb9d764bfc7c892784ceb Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 13 Nov 2026 11:45:28 +0100 Subject: [PATCH 3/4] refactor: explain the validation-only calls in input metadata creation --- .../storage_clients/_file_system/_key_value_store_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/apify/storage_clients/_file_system/_key_value_store_client.py b/src/apify/storage_clients/_file_system/_key_value_store_client.py index 27700480c..e3562a02f 100644 --- a/src/apify/storage_clients/_file_system/_key_value_store_client.py +++ b/src/apify/storage_clients/_file_system/_key_value_store_client.py @@ -125,7 +125,8 @@ async def _create_missing_metadata_for_input_file(key: str, record_path: Path) - logger.warning('Input file disappeared from path: "%s"', record_path) return - # Figure out the metadata from the file content + # Figure out the metadata from the file content. The parse and decode results are discarded: the calls only + # validate the content, so malformed input raises before any metadata is written. size = len(content) if record_path.suffix == '.json': json.loads(content.decode('utf-8')) From ea2811be7f683f826b8e3de20398369c5558b30d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 13 Nov 2026 11:45:29 +0100 Subject: [PATCH 4/4] test: cover input metadata for .txt and other file suffixes --- .../unit/storage_clients/test_file_system.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unit/storage_clients/test_file_system.py b/tests/unit/storage_clients/test_file_system.py index 0d930811a..8c6653d34 100644 --- a/tests/unit/storage_clients/test_file_system.py +++ b/tests/unit/storage_clients/test_file_system.py @@ -223,3 +223,28 @@ async def test_malformed_json_input_does_not_create_metadata() -> None: assert input_file.read_bytes() == b'{not json' assert not (kvs_path / f'INPUT.json.{METADATA_FILENAME}').exists() + + +@pytest.mark.parametrize( + ('input_file_name', 'expected_value', 'expected_content_type'), + [ + pytest.param('INPUT.txt', '42', 'text/plain; charset=utf-8', id='txt'), + pytest.param('INPUT.bin', b'42', 'application/octet-stream', id='other-suffix'), + ], +) +async def test_pre_existing_non_json_suffix_input_metadata( + input_file_name: str, expected_value: str | bytes, expected_content_type: str +) -> None: + """Input with a `.txt` or other non-JSON suffix gets text or binary metadata from its suffix.""" + configuration = Configuration.get_global_configuration() + kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default' + kvs_path.mkdir(parents=True) + (kvs_path / input_file_name).write_bytes(b'42') + + client = await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration) + record = await client.get_value(key=configuration.input_key) + + assert record is not None + assert record.value == expected_value + assert type(record.value) is type(expected_value) + assert record.content_type == expected_content_type