diff --git a/.importlinter b/.importlinter index 18812ecf..82cec34b 100644 --- a/.importlinter +++ b/.importlinter @@ -33,7 +33,7 @@ layers = progress ; Response-format conventions sit above the pure leaves because they read the ; code tables, and below every adapter that shapes a response with them. - _wqx + _wqx | _csv _response_metadata | codes | combining | interruptions | rdb credentials configuration diff --git a/NEWS.md b/NEWS.md index 7ce8bdf3..696cb632 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**09/10/2026:** **Bug fix:** `waterdata.get_samples()` and `waterdata.get_samples_summary()` now preserve leading zeros in code and identifier columns, including parameter codes, HUCs, FIPS codes, and monitoring-location and organization identifiers. These columns now contain strings instead of inferred numbers; measurement and count columns retain numeric inference. Samples and WQP share the existing two-pass CSV parsing convention. WQP behavior, request parameters, metadata, and datetime shaping are unchanged. + **09/01/2026:** **Announcement:** We at USGS Water Data for the Nation want your feedback! Tell us how we're doing by taking our quick [survey](https://usgswaterresources.gov1.qualtrics.com/jfe/form/SV_07gX8G1DeOtVrH8), available through September 2026. **08/27/2026:** **Bug fix:** `nwis.get_discharge_peaks` and `nwis.get_record(service='peaks')` discarded every peak whose date is only partly known. NWIS zero-fills the unknown part of a historical peak's date -- `YYYY-MM-00` when the day is not known, `YYYY-00-00` when the month is not either (the `Bd` and `Bm` `peak_cd` qualifiers) -- and neither parses as a date, so `preformat_peaks_response` coerced both to `NaT` and then dropped the row along with its discharge value. These are real peaks, and disproportionately a site's largest: site 14105700 lost 20 of its 167 peaks, among them an 1859 flood of 847,000 ft3/s, and 06934500 lost its 1844 peak of 700,000 ft3/s. Such a peak is now kept, with `datetime` left as `NaT`. The date is not completed into one NWIS does not have: a `datetime64` column cannot hold a partial date, so any value there would assert a day the record does not contain. **Behavior change:** peaks queries return more rows than before, and `datetime` may now be `NaT` -- a caller selecting on the datetime index will not see those peaks and should filter on `peak_dt` instead. **Behavior change:** `peak_dt` is no longer removed from the returned frame. It is the only column that holds a censored peak's year, since the peaks response has no `water_yr`, and the only dependable way to tell an unknown day from a known one -- `peak_cd` does not always include the qualifier (22 of 24 censored dates across six sites tested). For peaks with a resolved timestamp alongside explicit `year`/`month`/`day` and a `qualifier` field, use `waterdata.get_peaks()`. diff --git a/dataretrieval/_csv.py b/dataretrieval/_csv.py new file mode 100644 index 00000000..006653ec --- /dev/null +++ b/dataretrieval/_csv.py @@ -0,0 +1,36 @@ +"""Shared CSV parsing that preserves significant zeros in codes and identifiers.""" + +from collections.abc import Collection +from io import StringIO + +import pandas as pd +from pandas import DataFrame + + +def _is_code_column(name: str) -> bool: + """Report whether a column name denotes a code or identifier. + + Such columns (HUCs, parameter codes, FIPS codes) have leading zeros that + are significant and must be preserved as ``str``. A name qualifies if it + ends with "code" or contains "identifier", "huc", or "fips". + """ + lname = name.lower() + return lname.endswith("code") or any( + token in lname for token in ("identifier", "huc", "fips") + ) + + +def read_code_csv(text: str, *, infer_columns: Collection[str] = ()) -> DataFrame: + """Read CSV text with code/identifier columns as strings. + + Read the header first to select string columns before numeric inference can + discard leading zeros (``"00060"`` -> ``60``). Other columns retain pandas' + inferred types, and the default missing-value handling is unchanged. + ``infer_columns`` also retain inference when their names match a code or + identifier, allowing a caller to distinguish counts from identifiers. + """ + columns = pd.read_csv(StringIO(text), delimiter=",", nrows=0).columns + str_cols = { + col: str for col in columns if col not in infer_columns and _is_code_column(col) + } + return pd.read_csv(StringIO(text), delimiter=",", low_memory=False, dtype=str_cols) diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py index 6b07a5e4..1f292f39 100644 --- a/dataretrieval/waterdata/samples.py +++ b/dataretrieval/waterdata/samples.py @@ -11,13 +11,13 @@ import json import logging from collections.abc import Iterable -from io import StringIO from typing import Any, get_args from urllib.parse import quote import httpx import pandas as pd +from dataretrieval._csv import read_code_csv from dataretrieval._querying import to_str from dataretrieval._response_metadata import BaseMetadata from dataretrieval._validation import require_one_of @@ -100,7 +100,10 @@ def _get_samples_csv( **HTTPX_DEFAULTS, ) _raise_for_non_200(response) - df = pd.read_csv(StringIO(response.text), delimiter=",") + # This field counts alternate identifiers; it is not an identifier itself. + df = read_code_csv( + response.text, infer_columns=("AlternateLocation_IdentifierCount",) + ) return df, response @@ -307,7 +310,9 @@ def get_samples( Returns ------- df : ``pandas.DataFrame`` - Formatted data returned from the API query. For each + Formatted data returned from the API query. Code and identifier columns + retain leading zeros as strings; measurement columns retain inferred + numeric types. For each ``Date`` / ``Time`` / ``TimeZone`` triplet in the response (e.g. ``Activity_StartDate``, ``Activity_StartTime``, ``Activity_StartTimeZone``), an additional ``DateTime`` column @@ -403,7 +408,9 @@ def get_samples_summary( Returns ------- df : ``pandas.DataFrame`` - Formatted data returned from the API query. + Formatted data returned from the API query. Code and identifier columns + retain leading zeros as strings; count columns retain inferred numeric + types. md : :obj:`dataretrieval.utils.BaseMetadata` Custom ``dataretrieval`` metadata object pertaining to the query. diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 8538ffd1..b13fd0a5 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -12,12 +12,10 @@ import warnings from dataclasses import dataclass -from io import StringIO from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple -import pandas as pd - from dataretrieval import configuration as _configuration +from dataretrieval._csv import read_code_csv as _read_wqp_csv from dataretrieval._response_metadata import BaseMetadata from dataretrieval._validation import require_one_of from dataretrieval.configuration import ( @@ -120,34 +118,6 @@ def _resolve_profile( return kwargs -def _is_code_column(name: str) -> bool: - """Report whether a WQP column name denotes a code or identifier. - - Such columns (HUCs, parameter codes, FIPS codes) have leading zeros that - are significant and must be preserved as ``str``. A name qualifies if it - ends with "code" or contains "identifier", "huc", or "fips". - """ - lname = name.lower() - return lname.endswith("code") or any( - token in lname for token in ("identifier", "huc", "fips") - ) - - -def _read_wqp_csv(text: str) -> DataFrame: - """Read a WQP CSV, forcing code/identifier columns to ``str``. - - WQP returns codes with significant leading zeros — HUCs, parameter codes - (``USGSpcode``), FIPS state/county codes. A bare ``read_csv`` infers those as - int/float and drops the zeros without a warning (``"00060"`` -> ``60``, HUC8 - ``"07090002"`` -> ``7090002``). Read the header first, then re-read with - ``dtype=str`` for every column that :func:`_is_code_column` flags, so the zeros are - preserved. - """ - columns = pd.read_csv(StringIO(text), delimiter=",", nrows=0).columns - str_cols = {col: str for col in columns if _is_code_column(col)} - return pd.read_csv(StringIO(text), delimiter=",", low_memory=False, dtype=str_cols) - - def _query_wqp( service: str, *, diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 80154b47..5ecd6a0b 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -142,6 +142,74 @@ def test_mock_get_samples_summary(httpx_mock): assert md.comment is None +@pytest.mark.parametrize( + ("getter", "endpoint", "order"), + [ + ( + get_samples, + "results/fullphyschem?monitoringLocationIdentifier=USGS-00123&mimeType=text%2Fcsv", + [1, 0], + ), + (get_samples_summary, "summary/USGS-00123?mimeType=text%2Fcsv", [0, 1]), + ], +) +def test_samples_csv_preserves_identifiers(httpx_mock, getter, endpoint, order): + """Both Samples getters retain code text without changing value/date shaping.""" + request_url = "https://api.waterdata.usgs.gov/samples-data/" + endpoint + httpx_mock.add_response( + method="GET", + url=request_url, + headers={"mock_header": "value"}, + text=( + "USGSpcode,Location_HUCEightDigitCode,stateFips,countyFips," + "MonitoringLocationIdentifier,OrganizationIdentifier," + "ResultMeasureValue,resultCount,AlternateLocation_IdentifierCount,Activity_StartDate," + "Activity_StartTime,Activity_StartTimeZone\n" + "00060,07090002,01,003,00123,00007,1.5,2,2,2025-02-02,13:00:00,UTC\n" + "00065,07090003,02,005,00456,00008,,0,0,2025-01-01,09:00:00,UTC\n" + ), + ) + + df, md = getter(monitoring_location_id="USGS-00123") + + expected_codes = { + "USGSpcode": ["00060", "00065"], + "Location_HUCEightDigitCode": ["07090002", "07090003"], + "stateFips": ["01", "02"], + "countyFips": ["003", "005"], + "MonitoringLocationIdentifier": ["00123", "00456"], + "OrganizationIdentifier": ["00007", "00008"], + } + assert isinstance(df, DataFrame) + for column, values in expected_codes.items(): + assert df[column].tolist() == [values[i] for i in order] + assert pd.api.types.is_numeric_dtype(df["ResultMeasureValue"]) + assert pd.api.types.is_numeric_dtype(df["resultCount"]) + assert df["ResultMeasureValue"].iloc[order.index(0)] == 1.5 + assert pd.isna(df["ResultMeasureValue"].iloc[order.index(1)]) + assert df["resultCount"].tolist() == [[2, 0][i] for i in order] + assert pd.api.types.is_numeric_dtype(df["AlternateLocation_IdentifierCount"]) + assert df["AlternateLocation_IdentifierCount"].tolist() == [ + [2, 0][i] for i in order + ] + dates = ["2025-02-02", "2025-01-01"] + times = ["13:00:00", "09:00:00"] + assert df["Activity_StartDate"].tolist() == [dates[i] for i in order] + assert df["Activity_StartTime"].tolist() == [times[i] for i in order] + assert df["Activity_StartTimeZone"].tolist() == ["UTC", "UTC"] + if getter is get_samples: + assert df["Activity_StartDateTime"].tolist() == [ + pd.Timestamp("2025-01-01T09:00:00Z"), + pd.Timestamp("2025-02-02T13:00:00Z"), + ] + else: + assert "Activity_StartDateTime" not in df + assert md.url == request_url + assert isinstance(md.query_time, datetime.timedelta) + assert md.header.get("mock_header") == "value" + assert md.comment is None + + def test_get_samples_summary_rejects_list(): """The summary endpoint accepts only one site; a list must raise TypeError.""" with pytest.raises(TypeError, match="exactly one monitoring location"): diff --git a/tests/wqp_test.py b/tests/wqp_test.py index a2afda59..a9f9e9bb 100644 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -67,12 +67,15 @@ def test_read_wqp_csv_preserves_leading_zero_codes(): from dataretrieval.wqp import _read_wqp_csv csv = ( - "Location_HUCEightDigitCode,USGSpcode,ResultMeasureValue\n07090002,00060,1.5\n" + "Location_HUCEightDigitCode,USGSpcode,ResultMeasureValue," + "AlternateLocation_IdentifierCount\n07090002,00060,1.5,2\n" ) df = _read_wqp_csv(csv) assert df["Location_HUCEightDigitCode"].iloc[0] == "07090002" assert df["USGSpcode"].iloc[0] == "00060" assert df["ResultMeasureValue"].iloc[0] == 1.5 + # Preserve WQP's existing name-based inference, including count-like names. + assert df["AlternateLocation_IdentifierCount"].iloc[0] == "2" def test_get_results(httpx_mock):