Eurostat Data Import - Death by week and sex & Divorce indicators

statvar_imports/eurostat/Deaths_by_week_and_sex/Deaths_by_week_and_sex_metadata.csv

@@ -0,0 +1,2 @@
+parameter,value
+header_rows,1

statvar_imports/eurostat/Deaths_by_week_and_sex/Deaths_by_week_and_sex_pvmap.csv

@@ -0,0 +1,39 @@
+key,property1,value1,property2,value2,property3,value3,property4,value4
+
+# Global Dataflow properties (Standard Data Commons Mortality Modeling)
+"DATAFLOW:ESTAT:DEMO_R_MWK_TS(1.0)",populationType,dcs:MortalityEvent,measuredProperty,dcs:count,statType,dcs:measuredValue,,
+
+# --- AUTOMATED CLEANUP: Ignored Columns ---
+LAST UPDATE,#ignore,"",,,,,,
+CONF_STATUS,#ignore,"",,,,,,
+
+# Frequency dimension (P1W is standard for weekly duration)
+freq:W,observationPeriod,P1W,,,,,,
+
+# Sex / Gender dimension
+sex:T,gender,"",,,,,,
+sex:M,gender,dcs:Male,,,,,,
+sex:F,gender,dcs:Female,,,,,,
+
+# Geography dimension (Using direct passing)
+geo,observationAbout,{Data},,,,,,
+
+# Safely ignore the old week string so it doesn't cause errors
+TIME_PERIOD,#ignore,"",,,,,,
+
+# Map the new YYYY-MM-DD column directly
+CALENDAR_DATE,observationDate,{Data},,,,,,
+
+# Observation Value (FIXED: Restored the missing value mapping)
+OBS_VALUE,value,{Number},,,,,,
+
+# Safely Ignoring Transient Status Flags to Prevent Ingestion Warnings
+OBS_FLAG:p,#ignore,"",,,,,,
+OBS_FLAG:e,#ignore,"",,,,,,
+OBS_FLAG:b,#ignore,"",,,,,,
+OBS_FLAG:f,#ignore,"",,,,,,
+OBS_FLAG:n,#ignore,"",,,,,,
+OBS_FLAG:u,#ignore,"",,,,,,
+
+# Missing values
+OBS_FLAG:M,MissingValue,"",,,,,,

statvar_imports/eurostat/Deaths_by_week_and_sex/README.md

@@ -0,0 +1,74 @@
+# Eurostat Deaths By Week And Sex Import
+
+## Overview
+This dataset contains weekly death indicators at national and regional levels, sourced from Eurostat. The data tracks the total number of deaths broken down by sex and week across various European countries and regions.
+
+**type of place:** Country, NUTS Regions (Level 0-3)
+**years:** Historical data to present (2000-present, recorded weekly e.g., 2000-W01) 
+**place_resolution:** Resolved to DCIDs (e.g., dcid:country/FRA, dcid:nuts/AT113)
+
+## Data Source
+**Source URL:**
+https://ec.europa.eu/eurostat/databrowser/view/demo_r_mwk_ts/default/table?lang=en
+
+**Provenance Description:**
+The data is provided by Eurostat, the statistical office of the European Union. It belongs to the "Demography, population stock and balance" database under the "Deaths by week - special data collection" statistical theme, specifically the "Deaths by week and sex" (DEMO_R_MWK_TS) dataset.
+
+## Refresh Type
+Automatic Refresh
+
+The refresh is automated using the provided `run.sh` script, which handles both data download and processing.
+
+## How To Run Import
+To execute the complete import process (download and processing), run:
+./run.sh
+
+### Script Details:
+- **Download**: Uses `curl` to fetch the latest SDMX-CSV data from Eurostat's dissemination API.
+- **Processing**: Uses `stat_var_processor.py` to map raw data to Data Commons StatVarObservations using the PV map and metadata configuration.
+
+## Key Files
+- `run.sh`: Main execution script for download and processing.
+- `Deaths_by_week_and_sex_pvmap.csv`: Property-Value mapping for StatVar definitions and dimensions.
+- `Deaths_by_week_and_sex_metadata.csv`: Configuration parameters for the processor.
+- `places_resolved.csv`: Mapping of place codes to Data Commons DCIDs.
+- `Deaths_by_week_and_sex_output.csv`: Processed statistical observations.
+- `Deaths_by_week_and_sex_output.tmcf`: Template MCF mapping the CSV columns to Data Commons schema.
+
+## Validation
+To validate the generated data, use the Data Commons import tool (lint mode):
+```bash
+java -jar datacommons-import-tool.jar lint Deaths_by_week_and_sex_output.csv Deaths_by_week_and_sex_output.tmcf
+```
+The resulting reports (`report.json`, `summary_report.html`) in `dc_generated/` provide detailed insights into data quality and validation status.
+
+## Testing
+Testing is performed using the `test_data` directory:
+- Raw Input: `test_data/Deaths_by_week_and_sex_data_raw.csv`
+- Raw Processed Input: `test_data/Deaths_by_week_and_sex_data_raw_processed`
+- Expected Output: `test_data/Deaths_by_week_and_sex_output.csv`
+- Expected TMCF: `test_data/Deaths_by_week_and_sex_output.tmcf`
+
+## Run to Process the test data
+python3 ../../../tools/statvar_importer/stat_var_processor.py \
+  "--input_data=./test_data/Deaths_by_week_and_sex_data_raw_processed.csv" \
+  "--pv_map=./Deaths_by_week_and_sex_pvmap.csv" \
+  "--config_file=./Deaths_by_week_and_sex_metadata.csv" \
+  "--generate_statvar_name=True" \
+  "--skip_constant_csv_columns=False" \
+  "--output_columns=observationDate,observationAbout,variableMeasured,value,observationPeriod,measurementMethod,unit,scalingFactor" \
+  "--output_path=./final_output/Deaths_by_week_and_sex_output" \
+  "--places_resolved_csv=./places_resolved.csv" \
+  "--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf"
+
+## Run to Process the full data
+python3 ../../../tools/statvar_importer/stat_var_processor.py \
+  "--input_data=./input_files/Deaths_by_week_and_sex_data_raw_processed.csv" \
+  "--pv_map=./Deaths_by_week_and_sex_pvmap.csv" \
+  "--config_file=./Deaths_by_week_and_sex_metadata.csv" \
+  "--generate_statvar_name=True" \
+  "--skip_constant_csv_columns=False" \
+  "--output_columns=observationDate,observationAbout,variableMeasured,value,observationPeriod,measurementMethod,unit,scalingFactor" \
+  "--output_path=./final_output/Deaths_by_week_and_sex_output" \
+  "--places_resolved_csv=./places_resolved.csv" \
+  "--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf"

statvar_imports/eurostat/Deaths_by_week_and_sex/convert_week_to_date.py

@@ -0,0 +1,32 @@
+# Copyright 2026 Google LLC
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import pandas as pd
+import os
+from datetime import datetime
+
+# Get the absolute path of the directory where this script is located
+# (Which is now the Deaths_by_week_and_sex folder)
+script_dir = os.path.dirname(os.path.abspath(__file__))
+
+# Since the data is in the same folder as the script, we just append the filenames
+input_file = os.path.join(script_dir, 'Deaths_by_week_and_sex_data_raw.csv')
+output_file = os.path.join(script_dir, 'Deaths_by_week_and_sex_data_raw_processed.csv')
+
+# Load your raw data
+df = pd.read_csv(input_file)
+
+# Convert YYYY-Wxx to the Sunday of that week (ISO format: %G = Year, %V = Week, %u = Day of Week)
+df['CALENDAR_DATE'] = df['TIME_PERIOD'].apply(
+    lambda x: datetime.strptime(x + '-7', "%G-W%V-%u").strftime("%Y-%m-%d")
+)
+
+# Save the updated CSV to use in your importer
+df.to_csv(output_file, index=False)
+
+print(f"Data successfully processed and saved to:\n{output_file}")

statvar_imports/eurostat/Deaths_by_week_and_sex/manifest.json

@@ -0,0 +1,26 @@
+{
+  "import_specifications": [
+    {
+      "import_name": "EuroStat_Deaths_by_week_and_sex",
+      "curator_emails": [
+        "support@datacommons.org"
+      ],
+      "provenance_url": "https://ec.europa.eu/eurostat/databrowser/view/demo_r_mwk_ts/default/table?lang=en",
+      "provenance_description": "This data is a voluntary, granular collection of weekly deaths by sex, age, and region across European countries from 2000 onward, established by Eurostat to monitor mortality trends during and after the COVID-19 pandemic.",
+      "scripts": [
+        "run.sh"
+      ],
+      "source_files": [
+        "input_files/*.csv"
+      ],
+      "import_inputs": [
+        {
+          "template_mcf": "final_output/Deaths_by_week_and_sex_output.tmcf",
+          "cleaned_csv": "final_output/Deaths_by_week_and_sex_output.csv"
+        }
+      ],
+      "cron_schedule": "5 1 1,15 * *",
+      "resource_limits": {"cpu": 4, "memory": 8, "disk":100}
+        }
+   ]
+}