From a3f866059ee5ad067037b9f8f2d4f6a220a18855 Mon Sep 17 00:00:00 2001 From: Bhaskar Gurram Date: Sat, 14 Nov 2026 03:49:14 +0000 Subject: [PATCH] fix(waterdata): keep the parent statistic in statistics results _handle_nesting copied only parameter_code, unit_of_measure and parent_time_series_id from each data entry of a statistics feature, and dropped parent_statistic_id and parent_statistic_name. A parameter can have several daily series at one location (daily max, min and mean discharge at USGS-08049300, for example), each with its own normals and intervals, so get_stats_por and get_stats_date_range returned three medians for 07-01 with nothing but an opaque time series id to tell which daily statistic each one summarizes. Carry both fields through json_normalize's meta, as the R package does. --- NEWS.md | 2 ++ dataretrieval/waterdata/stats.py | 2 ++ tests/waterdata_test.py | 3 ++ tests/waterdata_utils_test.py | 48 ++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/NEWS.md b/NEWS.md index 11ce324a..549fd972 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**11/14/2026:** **Bug fix:** `waterdata.get_stats_por()` and `waterdata.get_stats_date_range()` now return the `parent_statistic_id` and `parent_statistic_name` columns that the Statistics API sends with each series. A parameter can have several daily series at one monitoring location, such as the daily maximum, minimum and mean of discharge, and each has its own statistics. The two columns were dropped, so the rows computed from the different daily series could not be told apart without looking up each `parent_time_series_id`. + **10/29/2026:** `waterdata.get_latest_continuous()` accepts `method_category`, a column USGS added to the `latest-continuous` collection in October 2026; `get_continuous()` has accepted it since 09/22/2026. It is the RLMS method category code (`STNRD`, `LMTUS`, `EXPER` or `UNKWN`) for the method in effect over an observation's interval, returned on every record and null for time series that have not been categorized. It could already be passed through `**queryables`; it is now a documented, keyword-only parameter. **10/16/2026:** **Bug fix:** the time-zone table behind the `*DateTime` columns that `waterdata.get_samples()` and `wqp.get_results()` append, and behind `utils.format_datetime()`, now follows the code lists those services use: the USGS NWIS time-zone list (Samples, and the USGS records that the Water Quality Portal passes through) and the WQX TimeZone domain (Water Quality Portal). USGS records write American Samoa as `ZP-11`, which was missing, so every timed American Samoa row from `get_samples()`, and every timed USGS American Samoa row from `get_results()`, had a `NaT` datetime; it is now UTC-11. 39 other codes from the two lists that used to become `NaT` now resolve, among them the remaining `ZP*` offsets, `IDLW`, `IDLE`, `NZT`, `SWT`, and the retired WQX codes `AHST` and `YST`. **Behavior change:** codes that the table read with a different meaning from the USGS list now take the USGS meaning: `CAST` (+9:30), `CCT` (+8), `EASST` (+11), `EAST` (+10), `NFT` (-3:30), `NZDT` (+13), `WAST` (+7), and `WAT` (-1). `FJST` is now UTC+13. Where the two lists disagree, WQX wins: `SST` stays American Samoa (UTC-11), the meaning the Water Quality Portal returns, and `BST` is UTC-11. diff --git a/dataretrieval/waterdata/stats.py b/dataretrieval/waterdata/stats.py index 0087641d..e3502869 100644 --- a/dataretrieval/waterdata/stats.py +++ b/dataretrieval/waterdata/stats.py @@ -82,6 +82,8 @@ def _handle_nesting( ["features", "properties", "data", "parameter_code"], ["features", "properties", "data", "unit_of_measure"], ["features", "properties", "data", "parent_time_series_id"], + ["features", "properties", "data", "parent_statistic_id"], + ["features", "properties", "data", "parent_statistic_name"], ], meta_prefix="", errors="ignore", diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 37078633..37544297 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -1845,6 +1845,9 @@ def test_get_stats_por(httpx_mock): assert len(df) == 11 assert df.loc[df["computation"] == "minimum", "percentile"].tolist() == [0.0] assert df.loc[df["computation"] == "arithmetic_mean", "percentile"].isna().all() + # Every row names the daily statistic its normals are computed from. + assert (df["parent_statistic_id"] == "00003").all() + assert (df["parent_statistic_name"] == "Mean").all() def test_get_stats_por_expanded_false(httpx_mock): diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 9348ffbe..7e1b3a9b 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -582,6 +582,54 @@ def test_handle_nesting_tolerates_missing_drop_columns(): assert df["monitoring_location_id"].iloc[0] == "USGS-12345" +def test_handle_nesting_keeps_parent_statistic(): + """A parameter can have several daily series (max, min, mean), each with + its own normals. The parent statistic is what tells those rows apart, so + it is kept beside ``parent_time_series_id``. + """ + body = { + "next": None, + "features": [ + { + "properties": { + "monitoring_location_id": "USGS-12345", + "data": [ + { + "parameter_code": "00060", + "unit_of_measure": "ft^3/s", + "parent_time_series_id": ts, + "parent_statistic_id": stat_id, + "parent_statistic_name": stat_name, + "values": [ + { + "time_of_year": "07-01", + "computation": "median", + "value": value, + } + ], + } + for ts, stat_id, stat_name, value in [ + ("ts-max", "00001", "Max", "373.5"), + ("ts-min", "00002", "Min", "267.0"), + ("ts-mean", "00003", "Mean", "320.0"), + ] + ], + }, + } + ], + } + + df = _handle_nesting(body, geopd=False) + + assert df[ + ["parent_statistic_id", "parent_statistic_name", "value"] + ].to_numpy().tolist() == [ + ["00001", "Max", "373.5"], + ["00002", "Min", "267.0"], + ["00003", "Mean", "320.0"], + ] + + def test_handle_nesting_returns_empty_on_empty_features(): """A mid-pagination empty page ({\"features\": [], \"next\": }) must not fail the downstream merge with