diff --git a/pysus/data/dbf_reader.py b/pysus/data/dbf_reader.py index 4730f48e..7b4b0fff 100644 --- a/pysus/data/dbf_reader.py +++ b/pysus/data/dbf_reader.py @@ -158,12 +158,14 @@ def read_dbf_fast( data = {} for fld in target: col: np.ndarray = records[fld.name] - decoded = np.empty(n, dtype=object) - for i in range(n): - val = col[i] - b = val if isinstance(val, bytes) else val.tobytes() - decoded[i] = _decode(b) - data[fld.name] = decoded + # Optimize decoding loop with list comprehension + data[fld.name] = [ + (b if isinstance(b, bytes) else b.tobytes()) + .decode(_ENCODING, errors="replace") + .replace("\x00", "") + .strip() + for b in col + ] return pd.DataFrame(data) @@ -273,12 +275,14 @@ def stream_dbf_fast( data = {} for fld in schema.fields: col: np.ndarray = records[fld.name] - decoded = np.empty(chunk_n, dtype=object) - for i in range(chunk_n): - val = col[i] - b = val if isinstance(val, bytes) else val.tobytes() - decoded[i] = _decode(b) - data[fld.name] = decoded + # Optimize decoding loop with list comprehension + data[fld.name] = [ + (b if isinstance(b, bytes) else b.tobytes()) + .decode(_ENCODING, errors="replace") + .replace("\x00", "") + .strip() + for b in col + ] yield pd.DataFrame(data)