Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,17 @@ df = get(
The API needs a key. Get a [free key](https://api.census.gov/data/key_signup.html) (it comes by email within minutes) and put it in `.env` as `CENSUS_API_KEY`.
If Gemini replies "This model is currently experiencing high demand", change the model in `scripts/gemini.py` to an older Flash model such as `gemini-3.5-flash`. The newest model is the busiest on the free tier.

### Backend: district measures and FastAPI

`backend/` fetches ACS block group data for language spoken at home, age 65+ and internet access, splits it across the six council districts by area, and writes `data/district-needs.csv` (columns in `data/README.md`).

```sh
uv run python -m backend.functions # rebuild data/district-needs.csv (needs CENSUS_API_KEY)
uv run uvicorn backend.api:app --reload # serve it at http://localhost:8000 (docs at /docs)
```

Endpoints: `GET /districts`, `GET /districts/{n}`, `GET /districts/geojson` (boundaries with measures), `GET /summary` (city-wide), `POST /refresh` (rebuild from the Census API).

The Census data source comes in a separate pull request: the BigQuery sandbox or the Census API. Merge one.

## Run it locally
Expand Down
79 changes: 79 additions & 0 deletions backend/api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
"""A FastAPI server for the district communication-needs data.

Run from the repo root with `uv run uvicorn backend.api:app --reload`, then open
http://localhost:8000/docs for the interactive docs.

It serves data/district-needs.csv, which backend/functions.py builds from the Census API.
POST /refresh rebuilds that file (it needs CENSUS_API_KEY in .env and takes a minute or so).
"""

import json

import pandas as pd
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware

from backend import functions

app = FastAPI(
title="Mableton communication needs",
description="Census measures of language, age 65+ and internet access by council district.",
)
# The data is public, so any frontend (including one running in the browser) may read it.
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET", "POST"])


def load() -> pd.DataFrame:
if not functions.OUTPUT.exists():
raise HTTPException(503, "No data yet: POST /refresh to build it.")
return pd.read_csv(functions.OUTPUT)


def records(df: pd.DataFrame) -> list[dict]:
# Through JSON so numpy numbers become plain ints and floats.
return json.loads(df.to_json(orient="records"))


@app.get("/districts")
def districts() -> list[dict]:
"""Every measure for each of the six council districts."""
return records(load())


@app.get("/districts/geojson")
def districts_geojson() -> dict:
"""The district boundaries, with every measure added to each district's properties."""
by_district = load().set_index("district")
geojson = json.loads(functions.DISTRICTS.read_text())
for feature in geojson["features"]:
district = feature["properties"]["district"]
measures = records(by_district.loc[[district]].reset_index())[0]
feature["properties"] = {
**feature["properties"],
**measures,
"census_2020_population": feature["properties"]["population"],
}
return geojson


@app.get("/districts/{district}")
def district(district: int) -> dict:
"""Every measure for one district (1–6)."""
rows = load().query("district == @district")
if rows.empty:
raise HTTPException(404, f"No district {district}; districts are 1–6.")
return records(rows)[0]


@app.get("/summary")
def summary() -> dict:
"""All of Mableton: the six districts' counts summed, with shares recomputed."""
return records(functions.city_totals(load()))[0]


@app.post("/refresh")
def refresh() -> dict:
"""Fetches fresh Census data and rebuilds data/district-needs.csv."""
table = functions.build()
table.to_csv(functions.OUTPUT, index=False)
return {"districts": len(table), "file": str(functions.OUTPUT.name)}
38 changes: 38 additions & 0 deletions backend/census.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
"""Fetches the raw inputs: ACS 5-year estimates and block group shapes for Cobb County.

Used by backend/functions.py, which runs the whole pipeline. The web API is backend/api.py.
"""

import io

import geopandas as gpd
import pandas as pd
import requests

from scripts.census_api import COUNTY, STATE, get

ACS = "2024/acs/acs5"
TIGER = "https://www2.census.gov/geo/tiger/TIGER2024/BG/tl_2024_{state}_bg.zip"
GEOGRAPHY = ["state", "county", "tract", "block group"]
MAX_VARIABLES = 49 # the API allows 50 per call, including the geography


def acs_block_groups(variables: list[str]) -> pd.DataFrame:
"""ACS estimates for every Cobb County block group, indexed by the 12-digit GEOID."""
frames = []
for i in range(0, len(variables), MAX_VARIABLES):
chunk = variables[i : i + MAX_VARIABLES]
df = get(ACS, chunk, "block group:*", f"state:{STATE} county:{COUNTY}")
df.index = df[GEOGRAPHY].agg("".join, axis=1).rename("GEOID")
frames.append(df[chunk])
estimates = pd.concat(frames, axis=1).apply(pd.to_numeric)
# Negative values are the API's codes for missing estimates (e.g. -666666666).
return estimates.mask(estimates < 0)


def block_group_shapes() -> gpd.GeoDataFrame:
"""TIGER/Line block group boundaries for Cobb County, with a GEOID column."""
response = requests.get(TIGER.format(state=STATE), timeout=300)
response.raise_for_status()
shapes = gpd.read_file(io.BytesIO(response.content), where=f"COUNTYFP='{COUNTY}'")
return shapes[["GEOID", "geometry"]]
129 changes: 129 additions & 0 deletions backend/functions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
"""Turns Census block group data into one row per Mableton council district.

Run from the repo root with `uv run python -m backend.functions`. It writes
data/district-needs.csv, which app.py can read (see data/README.md for the columns).

Block groups don't follow district lines, so each block group's counts are split across the
districts it overlaps in proportion to the share of its area inside each one (areal
weighting). That assumes people are spread evenly within a block group.
"""

from pathlib import Path

import geopandas as gpd
import pandas as pd

from backend import census

ROOT = Path(__file__).parent.parent
DISTRICTS = ROOT / "data" / "mableton-council-districts.geojson"
OUTPUT = ROOT / "data" / "district-needs.csv"
# UTM zone 16N, in meters, for measuring areas around Mableton
EQUAL_AREA = "EPSG:26916"


def lines(table: str, numbers) -> list[str]:
return [f"{table}_{n:03d}E" for n in numbers]


# B16004 lists, for each age group (5-17, 18-64, 65+), each non-English language followed by
# four lines on how well its speakers speak English: "very well", "well", "not well",
# "not at all". Limited English (the Census definition) is anything below "very well".
LANGUAGES = [4, 9, 14, 19, 26, 31, 36, 41, 48, 53, 58, 63]
LIMITED_ENGLISH = [n + k for n in LANGUAGES for k in (2, 3, 4)]

# Each count is the sum of these ACS variables.
COUNTS = {
# B01001 Sex by age
"population": lines("B01001", [1]),
"age_65_plus": lines("B01001", [*range(20, 26), *range(44, 50)]),
# B16004 Age by language spoken at home by ability to speak English (ages 5 and up)
"population_5_plus": lines("B16004", [1]),
"speak_only_english": lines("B16004", [3, 25, 47]),
"limited_english": lines("B16004", LIMITED_ENGLISH),
# B28002 Presence and types of internet subscriptions in household
"households": lines("B28002", [1]),
"households_no_internet": lines("B28002", [13]),
}

# Shares to compute after summing: name -> (numerator, denominator).
RATES = {
"non_english_pct": ("non_english", "population_5_plus"),
"limited_english_pct": ("limited_english", "population_5_plus"),
"age_65_plus_pct": ("age_65_plus", "population"),
"no_internet_pct": ("households_no_internet", "households"),
}


def block_group_counts(estimates: pd.DataFrame) -> pd.DataFrame:
"""Sums the ACS variables in COUNTS, one row per block group."""
return pd.DataFrame(
{name: estimates[variables].sum(axis=1) for name, variables in COUNTS.items()}
)


def district_weights(
shapes: gpd.GeoDataFrame, districts: gpd.GeoDataFrame
) -> pd.DataFrame:
"""For each block group and district they share: the share of the block group's area
inside that district. Columns GEOID, district, weight."""
shapes = shapes.to_crs(EQUAL_AREA)
shapes["bg_area"] = shapes.area
pieces = gpd.overlay(
shapes,
districts[["district", "geometry"]].to_crs(EQUAL_AREA),
how="intersection",
)
pieces["weight"] = pieces.area / pieces["bg_area"]
return pd.DataFrame(pieces[["GEOID", "district", "weight"]])


def apportion(counts: pd.DataFrame, weights: pd.DataFrame) -> pd.DataFrame:
"""Splits block group counts across districts by weight and sums them per district."""
joined = weights.join(counts, on="GEOID")
weighted = joined[counts.columns].mul(joined["weight"], axis=0)
return weighted.groupby(joined["district"]).sum().sort_index()


def add_rates(districts: pd.DataFrame) -> pd.DataFrame:
"""Adds the derived counts and the shares in RATES (0 to 1)."""
out = districts.copy()
out["non_english"] = out["population_5_plus"] - out["speak_only_english"]
for name, (numerator, denominator) in RATES.items():
out[name] = out[numerator] / out[denominator]
return out


def city_totals(table: pd.DataFrame) -> pd.DataFrame:
"""All six districts together, as one row: summed counts, with shares recomputed."""
totals = add_rates(table[list(COUNTS)].sum().to_frame().T)
totals[list(RATES)] = totals[list(RATES)].round(4)
return totals


def build() -> pd.DataFrame:
"""Fetches everything and returns one row per district: counts, then shares."""
estimates = census.acs_block_groups(
sorted({v for vs in COUNTS.values() for v in vs})
)
districts = gpd.read_file(DISTRICTS)
weights = district_weights(census.block_group_shapes(), districts)
table = add_rates(apportion(block_group_counts(estimates), weights))
counts = [c for c in table.columns if c not in RATES]
table[counts] = table[counts].round().astype(int)
table[list(RATES)] = table[list(RATES)].round(4)
return table.reset_index()


def main():
table = build()
table.to_csv(OUTPUT, index=False)
print(f"Wrote {OUTPUT.relative_to(ROOT)}")
# Sanity check: the areal estimate should land near each district's 2020 Census count.
census_2020 = gpd.read_file(DISTRICTS).set_index("district")["population"]
check = table.set_index("district")[["population"]].join(census_2020.rename("2020"))
print(check.to_string())


if __name__ == "__main__":
main()
22 changes: 21 additions & 1 deletion data/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,25 @@ Files here are committed and read by `app.py`.

- `mableton-council-districts.geojson`: the six council districts. Properties: `district` (1–6) and `population` (2020 Census).
- `mableton-city-limits.geojson`: the city boundary.
- `district-needs.csv`: communication-needs measures for each district, built by `uv run python -m backend.functions` (see below).

Both come from the public layer the city's [Find My District](https://www.mableton.gov/1247/Find-My-District) lookup uses: <https://services5.arcgis.com/Mm1BAVEiAXrYE9vn/arcgis/rest/services/CouncilDistricts_CityLimits/FeatureServer> (layers 2 and 3), converted to WGS 84 (EPSG:4326).
The two GeoJSON files come from the public layer the city's [Find My District](https://www.mableton.gov/1247/Find-My-District) lookup uses: <https://services5.arcgis.com/Mm1BAVEiAXrYE9vn/arcgis/rest/services/CouncilDistricts_CityLimits/FeatureServer> (layers 2 and 3), converted to WGS 84 (EPSG:4326).

## district-needs.csv

One row per council district (`district`, 1–6, joins to the GeoJSON). Source: ACS 2020–2024 5-year estimates (`2024/acs/acs5`) for Cobb County block groups. Block groups are split across the districts they overlap by share of area, so the numbers are estimates and won't exactly match the 2020 `population` in the GeoJSON.

Counts (people unless noted):

| Column | Meaning | ACS |
| --- | --- | --- |
| `population` | Total population | B01001_001 |
| `age_65_plus` | Age 65 and over | B01001 |
| `population_5_plus` | Age 5 and over (the universe for language) | B16004_001 |
| `speak_only_english` | Speak only English at home | B16004 |
| `non_english` | Speak a language other than English at home | B16004 |
| `limited_english` | Speak English less than "very well" | B16004 |
| `households` | Households | B28002_001 |
| `households_no_internet` | Households with no internet access at all | B28002_013 |

Shares (0 to 1): `non_english_pct` and `limited_english_pct` (of `population_5_plus`), `age_65_plus_pct` (of `population`), `no_internet_pct` (of `households`).
7 changes: 7 additions & 0 deletions data/district-needs.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
district,population,age_65_plus,population_5_plus,speak_only_english,limited_english,households,households_no_internet,non_english,non_english_pct,limited_english_pct,age_65_plus_pct,no_internet_pct
1,13062,1212,12129,8428,1867,4652,199,3701,0.3052,0.1539,0.0928,0.0427
2,13589,616,12921,7525,2846,4717,183,5396,0.4176,0.2203,0.0454,0.0388
3,13619,1942,12502,8972,1706,4678,123,3529,0.2823,0.1365,0.1426,0.0263
4,11898,1930,11223,9803,492,4823,93,1420,0.1265,0.0438,0.1622,0.0192
5,11471,1955,10989,8511,1052,4518,204,2478,0.2255,0.0958,0.1704,0.0451
6,13763,2109,12656,10397,620,5741,142,2259,0.1785,0.049,0.1532,0.0247
4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,17 @@ dependencies = [

[dependency-groups]
dev = [
"fastapi>=0.162.0",
"geopandas>=1.2.0",
"google-genai>=2.31.0",
"httpx>=0.28.1",
"pytest>=9.1.1",
"python-dotenv>=1.2.4",
"requests>=2.34.2",
"ruff>=0.17.1",
"uvicorn>=0.54.0",
]

[tool.pytest.ini_options]
testpaths = ["tests"]
pythonpath = ["."]
29 changes: 29 additions & 0 deletions tests/test_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
from fastapi.testclient import TestClient

from backend.api import app

client = TestClient(app)


def test_districts():
rows = client.get("/districts").json()
assert [row["district"] for row in rows] == [1, 2, 3, 4, 5, 6]


def test_one_district():
assert client.get("/districts/3").json()["district"] == 3
assert client.get("/districts/7").status_code == 404


def test_summary_adds_up():
rows = client.get("/districts").json()
summary = client.get("/summary").json()
assert summary["population"] == sum(row["population"] for row in rows)
assert 0 <= summary["non_english_pct"] <= 1


def test_geojson_has_measures():
feature = client.get("/districts/geojson").json()["features"][0]
assert {"district", "no_internet_pct", "census_2020_population"} <= set(
feature["properties"]
)
30 changes: 30 additions & 0 deletions tests/test_backend.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import geopandas as gpd
import pandas as pd
from shapely.geometry import box

from backend.functions import EQUAL_AREA, apportion, district_weights


def test_block_group_split_by_area():
# One block group straddles two districts, 3/4 in district 1 and 1/4 in district 2.
shapes = gpd.GeoDataFrame(
{"GEOID": ["a", "b"]},
geometry=[box(0, 0, 4, 1), box(4, 0, 5, 1)],
crs=EQUAL_AREA,
)
districts = gpd.GeoDataFrame(
{"district": [1, 2]},
geometry=[box(0, 0, 3, 1), box(3, 0, 5, 1)],
crs=EQUAL_AREA,
)
counts = pd.DataFrame({"population": [400, 100]}, index=pd.Index(["a", "b"]))

result = apportion(counts, district_weights(shapes, districts))

assert result.loc[1, "population"] == 300
assert result.loc[2, "population"] == 200


def test_district_needs_has_six_districts():
table = pd.read_csv("data/district-needs.csv")
assert sorted(table["district"]) == [1, 2, 3, 4, 5, 6]
Loading
Loading