diff --git a/src/apify/storage_clients/_file_system/_key_value_store_client.py b/src/apify/storage_clients/_file_system/_key_value_store_client.py index f8cd47a0..e3562a02 100644 --- a/src/apify/storage_clients/_file_system/_key_value_store_client.py +++ b/src/apify/storage_clients/_file_system/_key_value_store_client.py @@ -8,7 +8,7 @@ from typing_extensions import override from crawlee._consts import METADATA_FILENAME -from crawlee._utils.file import atomic_write, infer_mime_type, json_dumps +from crawlee._utils.file import atomic_write, json_dumps from crawlee.configuration import Configuration as CrawleeConfiguration from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata @@ -125,21 +125,24 @@ async def _create_missing_metadata_for_input_file(key: str, record_path: Path) - logger.warning('Input file disappeared from path: "%s"', record_path) return - # Figure out the metadata from the file content + # Figure out the metadata from the file content. The parse and decode results are discarded: the calls only + # validate the content, so malformed input raises before any metadata is written. size = len(content) if record_path.suffix == '.json': - value = json.loads(content.decode('utf-8')) + json.loads(content.decode('utf-8')) + content_type = 'application/json; charset=utf-8' elif record_path.suffix == '.txt': - value = content.decode('utf-8') + content.decode('utf-8') + content_type = 'text/plain; charset=utf-8' elif record_path.suffix == '': try: - value = json.loads(content.decode('utf-8')) - except json.JSONDecodeError: - value = content + json.loads(content.decode('utf-8')) + except (json.JSONDecodeError, UnicodeDecodeError): + content_type = 'application/octet-stream' + else: + content_type = 'application/json; charset=utf-8' else: - value = content - - content_type = infer_mime_type(value) + content_type = 'application/octet-stream' record_metadata = KeyValueStoreRecordMetadata(key=key, content_type=content_type, size=size) record_metadata_filepath = record_path.with_name(f'{record_path.name}.{METADATA_FILENAME}') diff --git a/tests/unit/storage_clients/test_file_system.py b/tests/unit/storage_clients/test_file_system.py index 6b30dfcb..8c6653d3 100644 --- a/tests/unit/storage_clients/test_file_system.py +++ b/tests/unit/storage_clients/test_file_system.py @@ -3,6 +3,7 @@ import asyncio import json from pathlib import Path +from typing import Any import pytest @@ -64,8 +65,23 @@ async def test_purge_preserves_input_file_and_metadata() -> None: assert json.loads(input_content) == json.loads('{"test": "input"}') -@pytest.mark.parametrize('input_file_name', ['INPUT', 'INPUT.json']) -async def test_pre_existing_input_used_by_actor(input_file_name: str) -> None: +@pytest.mark.parametrize( + 'input_file_name', + [pytest.param('INPUT', id='bare'), pytest.param('INPUT.json', id='json')], +) +@pytest.mark.parametrize( + 'pre_existing_input', + [ + pytest.param({'foo': 'bar'}, id='object'), + pytest.param('', id='empty-string'), + pytest.param(0, id='zero-integer'), + pytest.param(1.5, id='float'), + pytest.param(False, id='false'), + pytest.param(None, id='null'), + ], +) +async def test_pre_existing_input_used_by_actor(input_file_name: str, pre_existing_input: Any) -> None: + """Pre-existing JSON input preserves its value, type, bytes, and serialization metadata.""" configuration = Configuration() service_locator.set_configuration(configuration) @@ -74,15 +90,24 @@ async def test_pre_existing_input_used_by_actor(input_file_name: str) -> None: path_to_input.mkdir(parents=True) assert list(path_to_input.glob('*')) == [] - pre_existing_input = { - 'foo': 'bar', - } - - # Create pre-existing INPUT.json file - (path_to_input / input_file_name).write_text(json.dumps(pre_existing_input)) + input_file = path_to_input / input_file_name + input_bytes = json.dumps(pre_existing_input).encode('utf-8') + input_file.write_bytes(input_bytes) async with Actor(): - assert pre_existing_input == await Actor.get_input() + actual_input = await Actor.get_input() + assert actual_input == pre_existing_input + assert type(actual_input) is type(pre_existing_input) + store = await Actor.open_key_value_store() + stored_input = await store.get_value(configuration.input_key) + assert stored_input == pre_existing_input + assert type(stored_input) is type(pre_existing_input) + + assert input_file.read_bytes() == input_bytes + metadata = json.loads((path_to_input / f'{input_file_name}.{METADATA_FILENAME}').read_text()) + assert metadata['content_type'] == 'application/json; charset=utf-8' + assert metadata['key'] == configuration.input_key + assert metadata['size'] == len(input_bytes) # Make sure that the input file doesn't get renamed in the process and metadata are added assert set(path_to_input.glob('*')) == { @@ -92,6 +117,35 @@ async def test_pre_existing_input_used_by_actor(input_file_name: str) -> None: } +@pytest.mark.parametrize( + 'input_bytes', + [pytest.param(b'hello', id='plain'), pytest.param('hello'.encode('utf-16'), id='utf16')], +) +async def test_pre_existing_non_json_input_used_by_actor(input_bytes: bytes) -> None: + """Bare non-JSON input preserves its bytes and uses binary serialization metadata.""" + configuration = Configuration() + service_locator.set_configuration(configuration) + path_to_input = Path(configuration.storage_dir) / 'key_value_stores' / 'default' + path_to_input.mkdir(parents=True) + input_file = path_to_input / 'INPUT' + input_file.write_bytes(input_bytes) + + async with Actor(): + actual_input = await Actor.get_input() + assert actual_input == input_bytes + assert type(actual_input) is bytes + store = await Actor.open_key_value_store() + stored_input = await store.get_value(configuration.input_key) + assert stored_input == input_bytes + assert type(stored_input) is bytes + + assert input_file.read_bytes() == input_bytes + metadata = json.loads((path_to_input / f'INPUT.{METADATA_FILENAME}').read_text()) + assert metadata['content_type'] == 'application/octet-stream' + assert metadata['key'] == configuration.input_key + assert metadata['size'] == len(input_bytes) + + async def test_set_value_with_input_key_targets_existing_input_file() -> None: """`set_value` with the input key overwrites the existing `INPUT.json` instead of creating a duplicate.""" configuration = Configuration.get_global_configuration() @@ -132,3 +186,65 @@ async def test_record_exists_and_delete_value_target_existing_input_file() -> No await client.delete_value(key=configuration.input_key) assert await client.record_exists(key=configuration.input_key) is False assert not (kvs_path / 'INPUT.json').exists() + + +async def test_pre_existing_input_metadata_is_preserved() -> None: + """Opening an input record respects its explicit content type and leaves its metadata unchanged.""" + configuration = Configuration.get_global_configuration() + kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default' + kvs_path.mkdir(parents=True) + input_file = kvs_path / 'INPUT.json' + input_file.write_bytes(b'42') + metadata_file = kvs_path / f'INPUT.json.{METADATA_FILENAME}' + metadata_bytes = json.dumps( + {'key': configuration.input_key, 'content_type': 'text/plain; charset=utf-8', 'size': 2} + ).encode('utf-8') + metadata_file.write_bytes(metadata_bytes) + + client = await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration) + record = await client.get_value(key=configuration.input_key) + + assert record is not None + assert record.value == '42' + assert input_file.read_bytes() == b'42' + assert metadata_file.read_bytes() == metadata_bytes + + +async def test_malformed_json_input_does_not_create_metadata() -> None: + """Malformed JSON input raises without changing the input or creating record metadata.""" + configuration = Configuration.get_global_configuration() + kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default' + kvs_path.mkdir(parents=True) + input_file = kvs_path / 'INPUT.json' + input_file.write_bytes(b'{not json') + + with pytest.raises(json.JSONDecodeError): + await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration) + + assert input_file.read_bytes() == b'{not json' + assert not (kvs_path / f'INPUT.json.{METADATA_FILENAME}').exists() + + +@pytest.mark.parametrize( + ('input_file_name', 'expected_value', 'expected_content_type'), + [ + pytest.param('INPUT.txt', '42', 'text/plain; charset=utf-8', id='txt'), + pytest.param('INPUT.bin', b'42', 'application/octet-stream', id='other-suffix'), + ], +) +async def test_pre_existing_non_json_suffix_input_metadata( + input_file_name: str, expected_value: str | bytes, expected_content_type: str +) -> None: + """Input with a `.txt` or other non-JSON suffix gets text or binary metadata from its suffix.""" + configuration = Configuration.get_global_configuration() + kvs_path = Path(configuration.storage_dir) / 'key_value_stores' / 'default' + kvs_path.mkdir(parents=True) + (kvs_path / input_file_name).write_bytes(b'42') + + client = await ApifyFileSystemKeyValueStoreClient.open(id=None, name=None, alias=None, configuration=configuration) + record = await client.get_value(key=configuration.input_key) + + assert record is not None + assert record.value == expected_value + assert type(record.value) is type(expected_value) + assert record.content_type == expected_content_type