From c74367eb2a9d1c98be6da41888cfbf2298b960c6 Mon Sep 17 00:00:00 2001 From: Evan Murray Date: Wed, 28 Oct 2026 17:41:50 -0400 Subject: [PATCH 1/2] Create backend to get data from Census API --- README.md | 11 ++++ backend/api.py | 79 ++++++++++++++++++++++++ backend/census.py | 38 ++++++++++++ backend/functions.py | 129 ++++++++++++++++++++++++++++++++++++++++ data/README.md | 22 ++++++- data/district-needs.csv | 7 +++ pyproject.toml | 4 ++ uv.lock | 46 +++++++++++++- 8 files changed, 334 insertions(+), 2 deletions(-) create mode 100644 backend/api.py create mode 100644 backend/census.py create mode 100644 backend/functions.py create mode 100644 data/district-needs.csv diff --git a/README.md b/README.md index 4f8a946..fcb236c 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,17 @@ df = get( The API needs a key. Get a [free key](https://api.census.gov/data/key_signup.html) (it comes by email within minutes) and put it in `.env` as `CENSUS_API_KEY`. If Gemini replies "This model is currently experiencing high demand", change the model in `scripts/gemini.py` to an older Flash model such as `gemini-3.5-flash`. The newest model is the busiest on the free tier. +### Backend: district measures and FastAPI + +`backend/` fetches ACS block group data for language spoken at home, age 65+ and internet access, splits it across the six council districts by area, and writes `data/district-needs.csv` (columns in `data/README.md`). + +```sh +uv run python -m backend.functions # rebuild data/district-needs.csv (needs CENSUS_API_KEY) +uv run uvicorn backend.api:app --reload # serve it at http://localhost:8000 (docs at /docs) +``` + +Endpoints: `GET /districts`, `GET /districts/{n}`, `GET /districts/geojson` (boundaries with measures), `GET /summary` (city-wide), `POST /refresh` (rebuild from the Census API). + The Census data source comes in a separate pull request: the BigQuery sandbox or the Census API. Merge one. ## Run it locally diff --git a/backend/api.py b/backend/api.py new file mode 100644 index 0000000..e2d36cc --- /dev/null +++ b/backend/api.py @@ -0,0 +1,79 @@ +"""A FastAPI server for the district communication-needs data. + +Run from the repo root with `uv run uvicorn backend.api:app --reload`, then open +http://localhost:8000/docs for the interactive docs. + +It serves data/district-needs.csv, which backend/functions.py builds from the Census API. +POST /refresh rebuilds that file (it needs CENSUS_API_KEY in .env and takes a minute or so). +""" + +import json + +import pandas as pd +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware + +from backend import functions + +app = FastAPI( + title="Mableton communication needs", + description="Census measures of language, age 65+ and internet access by council district.", +) +# The data is public, so any frontend (including one running in the browser) may read it. +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET", "POST"]) + + +def load() -> pd.DataFrame: + if not functions.OUTPUT.exists(): + raise HTTPException(503, "No data yet: POST /refresh to build it.") + return pd.read_csv(functions.OUTPUT) + + +def records(df: pd.DataFrame) -> list[dict]: + # Through JSON so numpy numbers become plain ints and floats. + return json.loads(df.to_json(orient="records")) + + +@app.get("/districts") +def districts() -> list[dict]: + """Every measure for each of the six council districts.""" + return records(load()) + + +@app.get("/districts/geojson") +def districts_geojson() -> dict: + """The district boundaries, with every measure added to each district's properties.""" + by_district = load().set_index("district") + geojson = json.loads(functions.DISTRICTS.read_text()) + for feature in geojson["features"]: + district = feature["properties"]["district"] + measures = records(by_district.loc[[district]].reset_index())[0] + feature["properties"] = { + **feature["properties"], + **measures, + "census_2020_population": feature["properties"]["population"], + } + return geojson + + +@app.get("/districts/{district}") +def district(district: int) -> dict: + """Every measure for one district (1–6).""" + rows = load().query("district == @district") + if rows.empty: + raise HTTPException(404, f"No district {district}; districts are 1–6.") + return records(rows)[0] + + +@app.get("/summary") +def summary() -> dict: + """All of Mableton: the six districts' counts summed, with shares recomputed.""" + return records(functions.city_totals(load()))[0] + + +@app.post("/refresh") +def refresh() -> dict: + """Fetches fresh Census data and rebuilds data/district-needs.csv.""" + table = functions.build() + table.to_csv(functions.OUTPUT, index=False) + return {"districts": len(table), "file": str(functions.OUTPUT.name)} diff --git a/backend/census.py b/backend/census.py new file mode 100644 index 0000000..e65516b --- /dev/null +++ b/backend/census.py @@ -0,0 +1,38 @@ +"""Fetches the raw inputs: ACS 5-year estimates and block group shapes for Cobb County. + +Used by backend/functions.py, which runs the whole pipeline. The web API is backend/api.py. +""" + +import io + +import geopandas as gpd +import pandas as pd +import requests + +from scripts.census_api import COUNTY, STATE, get + +ACS = "2024/acs/acs5" +TIGER = "https://www2.census.gov/geo/tiger/TIGER2024/BG/tl_2024_{state}_bg.zip" +GEOGRAPHY = ["state", "county", "tract", "block group"] +MAX_VARIABLES = 49 # the API allows 50 per call, including the geography + + +def acs_block_groups(variables: list[str]) -> pd.DataFrame: + """ACS estimates for every Cobb County block group, indexed by the 12-digit GEOID.""" + frames = [] + for i in range(0, len(variables), MAX_VARIABLES): + chunk = variables[i : i + MAX_VARIABLES] + df = get(ACS, chunk, "block group:*", f"state:{STATE} county:{COUNTY}") + df.index = df[GEOGRAPHY].agg("".join, axis=1).rename("GEOID") + frames.append(df[chunk]) + estimates = pd.concat(frames, axis=1).apply(pd.to_numeric) + # Negative values are the API's codes for missing estimates (e.g. -666666666). + return estimates.mask(estimates < 0) + + +def block_group_shapes() -> gpd.GeoDataFrame: + """TIGER/Line block group boundaries for Cobb County, with a GEOID column.""" + response = requests.get(TIGER.format(state=STATE), timeout=300) + response.raise_for_status() + shapes = gpd.read_file(io.BytesIO(response.content), where=f"COUNTYFP='{COUNTY}'") + return shapes[["GEOID", "geometry"]] diff --git a/backend/functions.py b/backend/functions.py new file mode 100644 index 0000000..219d34e --- /dev/null +++ b/backend/functions.py @@ -0,0 +1,129 @@ +"""Turns Census block group data into one row per Mableton council district. + +Run from the repo root with `uv run python -m backend.functions`. It writes +data/district-needs.csv, which app.py can read (see data/README.md for the columns). + +Block groups don't follow district lines, so each block group's counts are split across the +districts it overlaps in proportion to the share of its area inside each one (areal +weighting). That assumes people are spread evenly within a block group. +""" + +from pathlib import Path + +import geopandas as gpd +import pandas as pd + +from backend import census + +ROOT = Path(__file__).parent.parent +DISTRICTS = ROOT / "data" / "mableton-council-districts.geojson" +OUTPUT = ROOT / "data" / "district-needs.csv" +# UTM zone 16N, in meters, for measuring areas around Mableton +EQUAL_AREA = "EPSG:26916" + + +def lines(table: str, numbers) -> list[str]: + return [f"{table}_{n:03d}E" for n in numbers] + + +# B16004 lists, for each age group (5-17, 18-64, 65+), each non-English language followed by +# four lines on how well its speakers speak English: "very well", "well", "not well", +# "not at all". Limited English (the Census definition) is anything below "very well". +LANGUAGES = [4, 9, 14, 19, 26, 31, 36, 41, 48, 53, 58, 63] +LIMITED_ENGLISH = [n + k for n in LANGUAGES for k in (2, 3, 4)] + +# Each count is the sum of these ACS variables. +COUNTS = { + # B01001 Sex by age + "population": lines("B01001", [1]), + "age_65_plus": lines("B01001", [*range(20, 26), *range(44, 50)]), + # B16004 Age by language spoken at home by ability to speak English (ages 5 and up) + "population_5_plus": lines("B16004", [1]), + "speak_only_english": lines("B16004", [3, 25, 47]), + "limited_english": lines("B16004", LIMITED_ENGLISH), + # B28002 Presence and types of internet subscriptions in household + "households": lines("B28002", [1]), + "households_no_internet": lines("B28002", [13]), +} + +# Shares to compute after summing: name -> (numerator, denominator). +RATES = { + "non_english_pct": ("non_english", "population_5_plus"), + "limited_english_pct": ("limited_english", "population_5_plus"), + "age_65_plus_pct": ("age_65_plus", "population"), + "no_internet_pct": ("households_no_internet", "households"), +} + + +def block_group_counts(estimates: pd.DataFrame) -> pd.DataFrame: + """Sums the ACS variables in COUNTS, one row per block group.""" + return pd.DataFrame( + {name: estimates[variables].sum(axis=1) for name, variables in COUNTS.items()} + ) + + +def district_weights( + shapes: gpd.GeoDataFrame, districts: gpd.GeoDataFrame +) -> pd.DataFrame: + """For each block group and district they share: the share of the block group's area + inside that district. Columns GEOID, district, weight.""" + shapes = shapes.to_crs(EQUAL_AREA) + shapes["bg_area"] = shapes.area + pieces = gpd.overlay( + shapes, + districts[["district", "geometry"]].to_crs(EQUAL_AREA), + how="intersection", + ) + pieces["weight"] = pieces.area / pieces["bg_area"] + return pd.DataFrame(pieces[["GEOID", "district", "weight"]]) + + +def apportion(counts: pd.DataFrame, weights: pd.DataFrame) -> pd.DataFrame: + """Splits block group counts across districts by weight and sums them per district.""" + joined = weights.join(counts, on="GEOID") + weighted = joined[counts.columns].mul(joined["weight"], axis=0) + return weighted.groupby(joined["district"]).sum().sort_index() + + +def add_rates(districts: pd.DataFrame) -> pd.DataFrame: + """Adds the derived counts and the shares in RATES (0 to 1).""" + out = districts.copy() + out["non_english"] = out["population_5_plus"] - out["speak_only_english"] + for name, (numerator, denominator) in RATES.items(): + out[name] = out[numerator] / out[denominator] + return out + + +def city_totals(table: pd.DataFrame) -> pd.DataFrame: + """All six districts together, as one row: summed counts, with shares recomputed.""" + totals = add_rates(table[list(COUNTS)].sum().to_frame().T) + totals[list(RATES)] = totals[list(RATES)].round(4) + return totals + + +def build() -> pd.DataFrame: + """Fetches everything and returns one row per district: counts, then shares.""" + estimates = census.acs_block_groups( + sorted({v for vs in COUNTS.values() for v in vs}) + ) + districts = gpd.read_file(DISTRICTS) + weights = district_weights(census.block_group_shapes(), districts) + table = add_rates(apportion(block_group_counts(estimates), weights)) + counts = [c for c in table.columns if c not in RATES] + table[counts] = table[counts].round().astype(int) + table[list(RATES)] = table[list(RATES)].round(4) + return table.reset_index() + + +def main(): + table = build() + table.to_csv(OUTPUT, index=False) + print(f"Wrote {OUTPUT.relative_to(ROOT)}") + # Sanity check: the areal estimate should land near each district's 2020 Census count. + census_2020 = gpd.read_file(DISTRICTS).set_index("district")["population"] + check = table.set_index("district")[["population"]].join(census_2020.rename("2020")) + print(check.to_string()) + + +if __name__ == "__main__": + main() diff --git a/data/README.md b/data/README.md index 68b4391..4b196c8 100644 --- a/data/README.md +++ b/data/README.md @@ -4,5 +4,25 @@ Files here are committed and read by `app.py`. - `mableton-council-districts.geojson`: the six council districts. Properties: `district` (1–6) and `population` (2020 Census). - `mableton-city-limits.geojson`: the city boundary. +- `district-needs.csv`: communication-needs measures for each district, built by `uv run python -m backend.functions` (see below). -Both come from the public layer the city's [Find My District](https://www.mableton.gov/1247/Find-My-District) lookup uses: (layers 2 and 3), converted to WGS 84 (EPSG:4326). +The two GeoJSON files come from the public layer the city's [Find My District](https://www.mableton.gov/1247/Find-My-District) lookup uses: (layers 2 and 3), converted to WGS 84 (EPSG:4326). + +## district-needs.csv + +One row per council district (`district`, 1–6, joins to the GeoJSON). Source: ACS 2020–2024 5-year estimates (`2024/acs/acs5`) for Cobb County block groups. Block groups are split across the districts they overlap by share of area, so the numbers are estimates and won't exactly match the 2020 `population` in the GeoJSON. + +Counts (people unless noted): + +| Column | Meaning | ACS | +| --- | --- | --- | +| `population` | Total population | B01001_001 | +| `age_65_plus` | Age 65 and over | B01001 | +| `population_5_plus` | Age 5 and over (the universe for language) | B16004_001 | +| `speak_only_english` | Speak only English at home | B16004 | +| `non_english` | Speak a language other than English at home | B16004 | +| `limited_english` | Speak English less than "very well" | B16004 | +| `households` | Households | B28002_001 | +| `households_no_internet` | Households with no internet access at all | B28002_013 | + +Shares (0 to 1): `non_english_pct` and `limited_english_pct` (of `population_5_plus`), `age_65_plus_pct` (of `population`), `no_internet_pct` (of `households`). diff --git a/data/district-needs.csv b/data/district-needs.csv new file mode 100644 index 0000000..19f69e5 --- /dev/null +++ b/data/district-needs.csv @@ -0,0 +1,7 @@ +district,population,age_65_plus,population_5_plus,speak_only_english,limited_english,households,households_no_internet,non_english,non_english_pct,limited_english_pct,age_65_plus_pct,no_internet_pct +1,13062,1212,12129,8428,1867,4652,199,3701,0.3052,0.1539,0.0928,0.0427 +2,13589,616,12921,7525,2846,4717,183,5396,0.4176,0.2203,0.0454,0.0388 +3,13619,1942,12502,8972,1706,4678,123,3529,0.2823,0.1365,0.1426,0.0263 +4,11898,1930,11223,9803,492,4823,93,1420,0.1265,0.0438,0.1622,0.0192 +5,11471,1955,10989,8511,1052,4518,204,2478,0.2255,0.0958,0.1704,0.0451 +6,13763,2109,12656,10397,620,5741,142,2259,0.1785,0.049,0.1532,0.0247 diff --git a/pyproject.toml b/pyproject.toml index cbf14ac..bf8b165 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,13 +11,17 @@ dependencies = [ [dependency-groups] dev = [ + "fastapi>=0.162.0", "geopandas>=1.2.0", "google-genai>=2.31.0", + "httpx>=0.28.1", "pytest>=9.1.1", "python-dotenv>=1.2.4", "requests>=2.34.2", "ruff>=0.17.1", + "uvicorn>=0.54.0", ] [tool.pytest.ini_options] testpaths = ["tests"] +pythonpath = ["."] diff --git a/uv.lock b/uv.lock index f63c908..f07d00d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -26,6 +26,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/b9/10a8bb13a0462e0bc4e02c0b7b8237dc24ab590f5bb4be94bec5e8f1d532/altair-6.3.0-py3-none-any.whl", hash = "sha256:7defb6ca730676dfc99a299768e2769f51585fcb3dc960ea71aacc368929d65e", size = 797192, upload-time = "2026-09-15T06:38:01.791Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + [[package]] name = "annotated-types" version = "0.8.0" @@ -312,12 +321,15 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "fastapi" }, { name = "geopandas" }, { name = "google-genai" }, + { name = "httpx" }, { name = "pytest" }, { name = "python-dotenv" }, { name = "requests" }, { name = "ruff" }, + { name = "uvicorn" }, ] [package.metadata] @@ -329,12 +341,15 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "fastapi", specifier = ">=0.162.0" }, { name = "geopandas", specifier = ">=1.2.0" }, { name = "google-genai", specifier = ">=2.31.0" }, + { name = "httpx", specifier = ">=0.28.1" }, { name = "pytest", specifier = ">=9.1.1" }, { name = "python-dotenv", specifier = ">=1.2.4" }, { name = "requests", specifier = ">=2.34.2" }, { name = "ruff", specifier = ">=0.17.1" }, + { name = "uvicorn", specifier = ">=0.54.0" }, ] [[package]] @@ -409,6 +424,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "fastapi" +version = "0.162.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/eb/2cf8c80abf09b2800aef0a83b41066fbfda8ee5495b7a08b9d1af40f7046/fastapi-0.162.0.tar.gz", hash = "sha256:6a22c1e2e6e32e0633ee7e170f00009a29b8228b16feb219ee42f32fd8d88a8f", size = 527867, upload-time = "2026-10-23T18:47:33.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/4f/3f2c25c9b43970b5e9a08beb5e04776247edea684ddc0930f4f02f0bb6cb/fastapi-0.162.0-py3-none-any.whl", hash = "sha256:3785963de305dd09912426cb81f2d230fce98ab3521cdfe97265c8d14983128c", size = 148912, upload-time = "2026-10-23T18:47:31.734Z" }, +] + [[package]] name = "geopandas" version = "1.2.0" @@ -763,6 +795,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/7f/c2d1b436b6e7cfebac140c2579a298344b85f2991a2ce5c3615cefb29400/numpy-2.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:7a14a461d9340f1b46b8648578aed9cdb8b3b018a8fac6c1dde2c9192a01a87f", size = 10883718, upload-time = "2026-10-10T20:05:28.547Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.45.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" }, +] + [[package]] name = "packaging" version = "26.3" From a36331cd4375fae06dd61604638f88ff3f056b31 Mon Sep 17 00:00:00 2001 From: Evan Murray Date: Wed, 28 Oct 2026 17:42:03 -0400 Subject: [PATCH 2/2] Create backend tests --- tests/test_api.py | 29 +++++++++++++++++++++++++++++ tests/test_backend.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/test_api.py create mode 100644 tests/test_backend.py diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..df35aa2 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,29 @@ +from fastapi.testclient import TestClient + +from backend.api import app + +client = TestClient(app) + + +def test_districts(): + rows = client.get("/districts").json() + assert [row["district"] for row in rows] == [1, 2, 3, 4, 5, 6] + + +def test_one_district(): + assert client.get("/districts/3").json()["district"] == 3 + assert client.get("/districts/7").status_code == 404 + + +def test_summary_adds_up(): + rows = client.get("/districts").json() + summary = client.get("/summary").json() + assert summary["population"] == sum(row["population"] for row in rows) + assert 0 <= summary["non_english_pct"] <= 1 + + +def test_geojson_has_measures(): + feature = client.get("/districts/geojson").json()["features"][0] + assert {"district", "no_internet_pct", "census_2020_population"} <= set( + feature["properties"] + ) diff --git a/tests/test_backend.py b/tests/test_backend.py new file mode 100644 index 0000000..c1bcafe --- /dev/null +++ b/tests/test_backend.py @@ -0,0 +1,30 @@ +import geopandas as gpd +import pandas as pd +from shapely.geometry import box + +from backend.functions import EQUAL_AREA, apportion, district_weights + + +def test_block_group_split_by_area(): + # One block group straddles two districts, 3/4 in district 1 and 1/4 in district 2. + shapes = gpd.GeoDataFrame( + {"GEOID": ["a", "b"]}, + geometry=[box(0, 0, 4, 1), box(4, 0, 5, 1)], + crs=EQUAL_AREA, + ) + districts = gpd.GeoDataFrame( + {"district": [1, 2]}, + geometry=[box(0, 0, 3, 1), box(3, 0, 5, 1)], + crs=EQUAL_AREA, + ) + counts = pd.DataFrame({"population": [400, 100]}, index=pd.Index(["a", "b"])) + + result = apportion(counts, district_weights(shapes, districts)) + + assert result.loc[1, "population"] == 300 + assert result.loc[2, "population"] == 200 + + +def test_district_needs_has_six_districts(): + table = pd.read_csv("data/district-needs.csv") + assert sorted(table["district"]) == [1, 2, 3, 4, 5, 6]