From 14dcd6844cd3d9a44ff51d0532f723ed728f3cc5 Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Thu, 3 Sep 2026 09:57:19 +0000 Subject: [PATCH 1/6] [Zurich] Update Zurich_Population and Zurich_Population_Number_Of_Company_Workplace_Employees - Update data sources to new City of Zurich Open Data endpoints (BEV324OD3240 and WIR255OD2552). - Add rollup aggregation script for wir_2552_wiki. - Update pvmap and metadata configs to map quarter-level records to authenticated Wikidata entities. - Update test data and templates. - Add automated validation configurations with deleted records threshold. - Update README.md and manifest.json. TAG=agy CONV=e14d0b1e-8efd-44a0-b3d1-5640746680a4 --- statvar_imports/zurich/README.md | 16 +- .../bev_3240_wiki/bev_3240_wiki_metadata.csv | 8 +- .../bev_3240_wiki/bev_3240_wiki_pvmap.csv | 41 +- .../test_data/bev_3240_wiki_input.csv | 76 ++-- ...zurich_population_bev_3240_wiki_output.csv | 26 ++ ...urich_population_bev_3240_wiki_output.tmcf | 12 +- .../bev_3240_wiki/validation_config.json | 13 + statvar_imports/zurich/manifest.json | 42 +- .../zurich/wir_2552_wiki/generate_rollups.py | 33 ++ .../test_data/wir_2552_wiki_input.csv | 76 ++-- .../zurich_population_common_output.tmcf | 9 - ...zurich_population_wir_2552_wiki_output.csv | 366 ++++++++++++++++-- ...urich_population_wir_2552_wiki_output.tmcf | 7 + ...ulation_wir_2552_wiki_output_stat_vars.mcf | 25 ++ .../wir_2552_wiki/validation_config.json | 13 + .../wir_2552_wiki/wir_2552_wiki_metadata.csv | 9 +- .../wir_2552_wiki/wir_2552_wiki_pvmap.csv | 66 +++- 17 files changed, 702 insertions(+), 136 deletions(-) create mode 100644 statvar_imports/zurich/bev_3240_wiki/validation_config.json create mode 100644 statvar_imports/zurich/wir_2552_wiki/generate_rollups.py delete mode 100644 statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_common_output.tmcf create mode 100644 statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.tmcf create mode 100644 statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output_stat_vars.mcf create mode 100644 statvar_imports/zurich/wir_2552_wiki/validation_config.json diff --git a/statvar_imports/zurich/README.md b/statvar_imports/zurich/README.md index 58df598a4b..8983f1ee04 100644 --- a/statvar_imports/zurich/README.md +++ b/statvar_imports/zurich/README.md @@ -1,13 +1,15 @@ 1. import_name": "Zurich_Population_Number_Of_Company_Workplace_Employees" 2. Import Overview -Zurich population data at Province and City Level. -Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/wir_2552_wiki.csv) +Number of companies, workplaces and employees in Zurich city at Province and City Level. +Source URL: [BFS WIR STATENT Data](https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552) Import Type: Fully Autorefresh Source Data Availability: 2011 to 2022 Release Frequency: P1Y -3. Preprocessing Steps (No) +3. Preprocessing Steps (Yes) +Generate rollups from the downloaded dataset: +python3 wir_2552_wiki/generate_rollups.py 4. Autorefresh Type @@ -15,7 +17,7 @@ Fully Autorefresh:"0 2 29 * * " (Runs at 2:00 AM on the 29th day of every month) 5. Script Execution Details -" python3 stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data='../../statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv' --pv_map='../../statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv' --config_file='../../statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv' --output_path=../../statvar_imports/zurich/wir_2552_wiki/output/zurich_population_wir_2552_wiki_output " +" python3 ../../util/download_util_script.py --download_url=https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552/download/WIR255OD2552.csv --output_folder=wir_2552_wiki/input_files && python3 wir_2552_wiki/generate_rollups.py && python3 ../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=wir_2552_wiki/input_files/WIR255OD2552_rollups.csv --pv_map=wir_2552_wiki/wir_2552_wiki_pvmap.csv --config_file=wir_2552_wiki/wir_2552_wiki_metadata.csv --output_columns=observationAbout,observationDate,value,variableMeasured --output_path=wir_2552_wiki/output/zurich_population_wir_2552_wiki --output_counters=wir_2552_wiki/counters/zurich_population_wir_2552_wiki_counters.csv " ##### @@ -45,8 +47,8 @@ Fully Autorefresh:" 0 6 29 * * " (Runs at 6:00 AM on the 29th of every month). 1. import_name": "Zurich_Population" 2. Import Overview -Zurich population data at Province and City Level. -Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3240_wiki.csv) +Total population of Zurich city by quarter and year at Province and City Level. +Source URL: [BEV324OD3240 Dataset](https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240) Import Type: Fully Autorefresh Source Data Availability: 1941 to 2023 Release Frequency: P1Y @@ -59,7 +61,7 @@ Fully Autorefresh:" 30 11 29 * * " (Runs at 11:30 AM on the 29th of every month) 5. Script Execution Details -" python3 stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data='../../statvar_imports/zurich/bev_3240_wiki/test_data/bev_3240_wiki_input.csv' --pv_map='../../statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_pvmap.csv' --config_file='../../statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv' --output_path=../../statvar_imports/zurich/bev_3240_wiki/output/zurich_population_bev_3240_wiki.csv_output " +" python3 ../../util/download_util_script.py --download_url=https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240/download/BEV324OD3240.csv --output_folder=bev_3240_wiki/input_files && python3 ../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=bev_3240_wiki/input_files/BEV324OD3240.csv --pv_map=bev_3240_wiki/bev_3240_wiki_pvmap.csv --config_file=bev_3240_wiki/bev_3240_wiki_metadata.csv --output_columns=observationAbout,observationDate,value,variableMeasured --output_path=bev_3240_wiki/output/zurich_population_bev_3240_wiki --output_counters=bev_3240_wiki/counters/zurich_population_bev_3240_wiki_counters.csv " ##### diff --git a/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv b/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv index 79237e739e..3db1541727 100644 --- a/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv +++ b/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv @@ -1,5 +1,5 @@ parameter,value -url,https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3240_wiki.csv +url,https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240 description, place_type,Country #places_within, @@ -8,7 +8,7 @@ end_date, release_frequency, process, comments, -output_columns,"observationAbout,observationDate,value,variableMeasured,unit" +output_columns,"observationAbout,observationDate,value,variableMeasured" header_rows,1 -#input_rows,10 -#Import_Name,Zurich_Population +reconcile_nodes,False +output_counters,bev_3240_wiki/counters/zurich_population_bev_3240_wiki_counters.csv diff --git a/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_pvmap.csv b/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_pvmap.csv index 72007e6c3a..0979cb1820 100644 --- a/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_pvmap.csv +++ b/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_pvmap.csv @@ -1,4 +1,37 @@ -key,prop,val,p1,v1 -StichtagDatJahr,observationDate,{Number},, -AnzBestWir,value,{Number},populationType,Person -wikiId,observationAbout,wikidataId/{Data},, \ No newline at end of file +key,prop,val,p1,v1 +StichtagDatJahr,observationDate,{Number},, +AnzBestWir,value,{Number},populationType,Person +QuarLang:Rathaus,observationAbout,wikidataId/Q692511,, +QuarLang:Hochschulen,observationAbout,wikidataId/Q39240,, +QuarLang:Lindenhof,observationAbout,wikidataId/Q10987378,, +QuarLang:City,observationAbout,wikidataId/Q1093831,, +QuarLang:Wollishofen,observationAbout,wikidataId/Q642353,, +QuarLang:Leimbach,observationAbout,wikidataId/Q691367,, +QuarLang:Enge,observationAbout,wikidataId/Q648218,, +QuarLang:Alt-Wiedikon,observationAbout,wikidataId/Q433012,, +QuarLang:Friesenberg,observationAbout,wikidataId/Q693357,, +QuarLang:Sihlfeld,observationAbout,wikidataId/Q370104,, +QuarLang:Werd,observationAbout,wikidataId/Q531899,, +QuarLang:Langstrasse,observationAbout,wikidataId/Q1805410,, +QuarLang:Hard,observationAbout,wikidataId/Q870084,, +QuarLang:Gewerbeschule,observationAbout,wikidataId/Q693413,, +QuarLang:Escher Wyss,observationAbout,wikidataId/Q687052,, +QuarLang:Unterstrass,observationAbout,wikidataId/Q656446,, +QuarLang:Oberstrass,observationAbout,wikidataId/Q693483,, +QuarLang:Fluntern,observationAbout,wikidataId/Q693269,, +QuarLang:Hottingen,observationAbout,wikidataId/Q693454,, +QuarLang:Hirslanden,observationAbout,wikidataId/Q476940,, +QuarLang:Witikon,observationAbout,wikidataId/Q392079,, +QuarLang:Seefeld,observationAbout,wikidataId/Q692773,, +QuarLang:Mühlebach,observationAbout,wikidataId/Q693397,, +QuarLang:Weinegg,observationAbout,wikidataId/Q693321,, +QuarLang:Albisrieden,observationAbout,wikidataId/Q80797,, +QuarLang:Altstetten,observationAbout,wikidataId/Q445711,, +QuarLang:Höngg,observationAbout,wikidataId/Q455496,, +QuarLang:Wipkingen,observationAbout,wikidataId/Q678030,, +QuarLang:Affoltern,observationAbout,wikidataId/Q382903,, +QuarLang:Oerlikon,observationAbout,wikidataId/Q167179,, +QuarLang:Seebach,observationAbout,wikidataId/Q276792,, +QuarLang:Saatlen,observationAbout,wikidataId/Q652455,, +QuarLang:Schwamendingen-Mitte,observationAbout,wikidataId/Q692728,, +QuarLang:Hirzenbach,observationAbout,wikidataId/Q693374,, \ No newline at end of file diff --git a/statvar_imports/zurich/bev_3240_wiki/test_data/bev_3240_wiki_input.csv b/statvar_imports/zurich/bev_3240_wiki/test_data/bev_3240_wiki_input.csv index 48bb92542b..fb8dba4656 100644 --- a/statvar_imports/zurich/bev_3240_wiki/test_data/bev_3240_wiki_input.csv +++ b/statvar_imports/zurich/bev_3240_wiki/test_data/bev_3240_wiki_input.csv @@ -1,25 +1,51 @@ -StichtagDatJahr,AnzBestWir,wikiId -2023,3343,Q692511 -2022,3330,Q692511 -2021,3275,Q692511 -2020,3317,Q692511 -2019,3307,Q692511 -2018,3267,Q692511 -2017,3266,Q692511 -2016,3226,Q692511 -2015,3197,Q692511 -2014,3236,Q692511 -2013,3194,Q692511 -2012,3164,Q692511 -2011,3140,Q692511 -2010,3099,Q692511 -2009,3110,Q692511 -2008,3051,Q692511 -2007,3079,Q692511 -2006,3097,Q692511 -2005,3081,Q692511 -2004,3115,Q692511 -2003,3149,Q692511 -2002,3128,Q692511 -2001,3103,Q692511 -2000,3109,Q692511 +StichtagDatJahr,QuarSort,QuarLang,AnzBestWir +2025,11,Rathaus,3252 +2024,11,Rathaus,3260 +2023,11,Rathaus,3343 +2022,11,Rathaus,3330 +2021,11,Rathaus,3275 +2020,11,Rathaus,3317 +2019,11,Rathaus,3307 +2018,11,Rathaus,3267 +2017,11,Rathaus,3266 +2016,11,Rathaus,3226 +2015,11,Rathaus,3197 +2014,11,Rathaus,3236 +2013,11,Rathaus,3194 +2012,11,Rathaus,3164 +2011,11,Rathaus,3140 +2010,11,Rathaus,3099 +2009,11,Rathaus,3110 +2008,11,Rathaus,3051 +2007,11,Rathaus,3079 +2006,11,Rathaus,3097 +2005,11,Rathaus,3081 +2004,11,Rathaus,3115 +2003,11,Rathaus,3149 +2002,11,Rathaus,3128 +2001,11,Rathaus,3103 +2000,11,Rathaus,3109 +1999,11,Rathaus,3104 +1998,11,Rathaus,3050 +1997,11,Rathaus,3083 +1996,11,Rathaus,3078 +1995,11,Rathaus,3049 +1994,11,Rathaus,3089 +1993,11,Rathaus,3133 +1992,11,Rathaus,3186 +1991,11,Rathaus,3173 +1990,11,Rathaus,3145 +1989,11,Rathaus,3161 +1988,11,Rathaus,3187 +1987,11,Rathaus,3218 +1986,11,Rathaus,3177 +1985,11,Rathaus,3181 +1984,11,Rathaus,3277 +1983,11,Rathaus,3304 +1982,11,Rathaus,3356 +1981,11,Rathaus,3501 +1980,11,Rathaus,3539 +1979,11,Rathaus,3551 +1978,11,Rathaus,3710 +1977,11,Rathaus,3802 +1976,11,Rathaus,3836 diff --git a/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv b/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv index af72c49c35..42048306f6 100644 --- a/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv +++ b/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv @@ -1,4 +1,6 @@ observationAbout,observationDate,value,variableMeasured,unit +wikidataId/Q692511,2025,3252,dcid:Count_Person, +wikidataId/Q692511,2024,3260,dcid:Count_Person, wikidataId/Q692511,2023,3343,dcid:Count_Person, wikidataId/Q692511,2022,3330,dcid:Count_Person, wikidataId/Q692511,2021,3275,dcid:Count_Person, @@ -23,3 +25,27 @@ wikidataId/Q692511,2003,3149,dcid:Count_Person, wikidataId/Q692511,2002,3128,dcid:Count_Person, wikidataId/Q692511,2001,3103,dcid:Count_Person, wikidataId/Q692511,2000,3109,dcid:Count_Person, +wikidataId/Q692511,1999,3104,dcid:Count_Person, +wikidataId/Q692511,1998,3050,dcid:Count_Person, +wikidataId/Q692511,1997,3083,dcid:Count_Person, +wikidataId/Q692511,1996,3078,dcid:Count_Person, +wikidataId/Q692511,1995,3049,dcid:Count_Person, +wikidataId/Q692511,1994,3089,dcid:Count_Person, +wikidataId/Q692511,1993,3133,dcid:Count_Person, +wikidataId/Q692511,1992,3186,dcid:Count_Person, +wikidataId/Q692511,1991,3173,dcid:Count_Person, +wikidataId/Q692511,1990,3145,dcid:Count_Person, +wikidataId/Q692511,1989,3161,dcid:Count_Person, +wikidataId/Q692511,1988,3187,dcid:Count_Person, +wikidataId/Q692511,1987,3218,dcid:Count_Person, +wikidataId/Q692511,1986,3177,dcid:Count_Person, +wikidataId/Q692511,1985,3181,dcid:Count_Person, +wikidataId/Q692511,1984,3277,dcid:Count_Person, +wikidataId/Q692511,1983,3304,dcid:Count_Person, +wikidataId/Q692511,1982,3356,dcid:Count_Person, +wikidataId/Q692511,1981,3501,dcid:Count_Person, +wikidataId/Q692511,1980,3539,dcid:Count_Person, +wikidataId/Q692511,1979,3551,dcid:Count_Person, +wikidataId/Q692511,1978,3710,dcid:Count_Person, +wikidataId/Q692511,1977,3802,dcid:Count_Person, +wikidataId/Q692511,1976,3836,dcid:Count_Person, diff --git a/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf b/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf index 2cf36b0678..73edb97d01 100644 --- a/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf +++ b/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf @@ -1,7 +1,7 @@ -Node: E:bev_3240_wiki_output->E0 -observationAbout: C:bev_3240_wiki_output->observationAbout -observationDate: C:bev_3240_wiki_output->observationDate -value: C:bev_3240_wiki_output->value -variableMeasured: C:bev_3240_wiki_output->variableMeasured -unit: C:bev_3240_wiki_output->unit +Node: E:zurich_population_bev_3240_wiki_output->E0 +observationAbout: C:zurich_population_bev_3240_wiki_output->observationAbout +observationDate: C:zurich_population_bev_3240_wiki_output->observationDate +value: C:zurich_population_bev_3240_wiki_output->value +variableMeasured: C:zurich_population_bev_3240_wiki_output->variableMeasured +unit: C:zurich_population_bev_3240_wiki_output->unit typeOf: dcs:StatVarObservation diff --git a/statvar_imports/zurich/bev_3240_wiki/validation_config.json b/statvar_imports/zurich/bev_3240_wiki/validation_config.json new file mode 100644 index 0000000000..66014b3369 --- /dev/null +++ b/statvar_imports/zurich/bev_3240_wiki/validation_config.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + } + ] +} diff --git a/statvar_imports/zurich/manifest.json b/statvar_imports/zurich/manifest.json index 9ec063ffb1..0ca7ee230f 100644 --- a/statvar_imports/zurich/manifest.json +++ b/statvar_imports/zurich/manifest.json @@ -5,22 +5,31 @@ "curator_emails": [ "support@datacommons.org" ], - "provenance_url": "https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/wir_2552_wiki.csv", - "provenance_description": "Number of companys, workplaces and employees of Zurich city contains structure and organization of the city administration", + "provenance_url": "https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552", + "provenance_description": "Number of companies, workplaces and employees of Zurich city contains structure and organization of the city administration", "scripts": [ - "../../util/download_util_script.py --download_url=https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/wir_2552_wiki.csv --output_folder=wir_2552_wiki/input_files", - "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=wir_2552_wiki/input_files/wir_2552_wiki.csv --pv_map=wir_2552_wiki/wir_2552_wiki_pvmap.csv --config_file=wir_2552_wiki/wir_2552_wiki_metadata.csv --output_path=wir_2552_wiki/output/zurich_population_wir_2552_wiki" + "../../util/download_util_script.py --download_url=https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552/download/WIR255OD2552.csv --output_folder=wir_2552_wiki/input_files", + "wir_2552_wiki/generate_rollups.py", + "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=wir_2552_wiki/input_files/WIR255OD2552_rollups.csv --pv_map=wir_2552_wiki/wir_2552_wiki_pvmap.csv --config_file=wir_2552_wiki/wir_2552_wiki_metadata.csv --output_columns=observationAbout,observationDate,value,variableMeasured --output_path=wir_2552_wiki/output/zurich_population_wir_2552_wiki --output_counters=wir_2552_wiki/counters/zurich_population_wir_2552_wiki_counters.csv" ], "source_files": [ - "wir_2552_wiki/input_files/wir_2552_wiki.csv" + "wir_2552_wiki/input_files/WIR255OD2552_rollups.csv", + "wir_2552_wiki/counters/*.csv" ], "import_inputs": [ { "template_mcf": "wir_2552_wiki/output/zurich_population_wir_2552_wiki.tmcf", - "cleaned_csv": "wir_2552_wiki/output/zurich_population_wir_2552_wiki.csv" + "cleaned_csv": "wir_2552_wiki/output/zurich_population_wir_2552_wiki.csv", + "node_mcf": "wir_2552_wiki/output/zurich_population_wir_2552_wiki_stat_vars.mcf" } ], - "cron_schedule": "0 2 29 * *" + "resource_limits": { + "cpu": 8, + "memory": 64, + "disk": 100 + }, + "cron_schedule": "0 2 29 * *", + "validation_config_file": "wir_2552_wiki/validation_config.json" }, { "import_name": "Zurich_Population_By_Age", @@ -49,14 +58,15 @@ "curator_emails": [ "support@datacommons.org" ], - "provenance_url": "https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3240_wiki.csv", + "provenance_url": "https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240", "provenance_description": "Population of Zurich city contains structure and organization of the city administration", "scripts": [ - "../../util/download_util_script.py --download_url=https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3240_wiki.csv --output_folder=bev_3240_wiki/input_files", - "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=bev_3240_wiki/input_files/bev_3240_wiki.csv --pv_map=bev_3240_wiki/bev_3240_wiki_pvmap.csv --config_file=bev_3240_wiki/bev_3240_wiki_metadata.csv --output_path=bev_3240_wiki/output/zurich_population_bev_3240_wiki" + "../../util/download_util_script.py --download_url=https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240/download/BEV324OD3240.csv --output_folder=bev_3240_wiki/input_files", + "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=bev_3240_wiki/input_files/BEV324OD3240.csv --pv_map=bev_3240_wiki/bev_3240_wiki_pvmap.csv --config_file=bev_3240_wiki/bev_3240_wiki_metadata.csv --output_columns=observationAbout,observationDate,value,variableMeasured --output_path=bev_3240_wiki/output/zurich_population_bev_3240_wiki --output_counters=bev_3240_wiki/counters/zurich_population_bev_3240_wiki_counters.csv" ], "source_files": [ - "bev_3240_wiki/input_files/bev_3240_wiki.csv" + "bev_3240_wiki/input_files/BEV324OD3240.csv", + "bev_3240_wiki/counters/*.csv" ], "import_inputs": [ { @@ -64,7 +74,13 @@ "cleaned_csv": "bev_3240_wiki/output/zurich_population_bev_3240_wiki.csv" } ], - "cron_schedule": "30 11 29 * *" + "resource_limits": { + "cpu": 8, + "memory": 64, + "disk": 100 + }, + "cron_schedule": "30 11 29 * *", + "validation_config_file": "bev_3240_wiki/validation_config.json" }, { "import_name": "Zurich_Population_Number_Of_Birth_By_Origin", @@ -178,4 +194,4 @@ "cron_schedule": "55 10 29 * *" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py b/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py new file mode 100644 index 0000000000..402dd933c2 --- /dev/null +++ b/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Preprocesses Zurich WIR255OD2552 dataset to filter total roll-up rows (Alle Rechtsformen and Alle Betriebsgrössen).""" + +import os +import pandas as pd + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +INPUT_RAW = os.path.join(CURRENT_DIR, 'input_files', 'WIR255OD2552.csv') +OUTPUT_ROLLUPS = os.path.join(CURRENT_DIR, 'input_files', 'WIR255OD2552_rollups.csv') + +def main(): + if not os.path.exists(INPUT_RAW): + raise FileNotFoundError(f"{INPUT_RAW} not found.") + + try: + df = pd.read_csv(INPUT_RAW, encoding='utf-8') + except UnicodeDecodeError: + df = pd.read_csv(INPUT_RAW, encoding='iso-8859-1') + + # Keep only the total rows (Alle Rechtsformen and Alle Betriebsgrössen) + df_rollups = df[(df['RechtsformSort'] == 0) & (df['BetriebsgrSort'] == 0)].copy() + + # Convert numeric columns, converting non-numeric markers (e.g. 'K') to NaN/numeric + val_cols = ['Arbeitsstaetten', 'AnzBesch', 'AnzBeschW', 'AnzBeschM', 'AnzVZA', 'AnzVZAW', 'AnzVZAM'] + for col in val_cols: + if col in df_rollups.columns: + df_rollups[col] = pd.to_numeric(df_rollups[col], errors='coerce') + + df_rollups.to_csv(OUTPUT_ROLLUPS, index=False, encoding='utf-8') + print(f"Successfully generated {OUTPUT_ROLLUPS} with {len(df_rollups)} rows.") + +if __name__ == '__main__': + main() diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv b/statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv index 26640c54a0..d13f6706ba 100644 --- a/statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv +++ b/statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv @@ -1,25 +1,51 @@ -Zeit,companies,employees,employees_f,employees_m,fte,fte_f,fte_m,wikiId -2011,1522,10180,5492,4688,7205,3493,3710,Q692511 -2011,700,13203,7025,6178,9090,4376,4714,Q39240 -2011,1193,15160,7649,7511,12213,5602,6608,Q10987378 -2011,1834,30084,14496,15588,24633,10861,13768,Q1093831 -2011,1050,7578,3396,4182,5865,2271,3594,Q642353 -2011,177,640,340,300,460,217,246,Q691367 -2011,2463,34077,15482,18595,26297,10298,16002,Q648218 -2011,1668,25737,10281,15456,20428,6962,13465,Q433012 -2011,358,4510,3113,1397,3523,2311,1209,Q693357 -2011,1690,11292,5100,6192,8196,3148,5049,Q370104 -2011,984,9023,4202,4821,6753,2834,3919,Q531899 -2011,2066,19957,8639,11318,15238,5743,9495,Q1805410 -2011,842,7081,2376,4705,5497,1455,4045,Q870084 -2011,1363,15655,7201,8454,11596,4844,6753,Q693413 -2011,1189,28501,9554,18947,24293,7144,17146,Q687052 -2011,1931,15235,7380,7855,11198,4786,6413,Q656446 -2011,1030,10501,5217,5284,7210,3253,3956,Q693483 -2011,838,14560,8961,5599,11196,6607,4590,Q693269 -2011,1701,11377,6669,4708,8268,4448,3822,Q693454 -2011,715,3824,2212,1612,2678,1383,1296,Q476940 -2011,504,2042,1257,785,1483,833,647,Q392079 -2011,1526,9963,4975,4988,7537,3393,4146,Q692773 -2011,1503,9161,4556,4605,6891,3024,3869,Q693397 -2011,646,9788,5820,3968,6925,3913,3011,Q693321 +Jahr,RaumSort,RaumLang,RechtsformSort,RechtsformLang,BetriebsgrSort,BetriebsgrLang,Arbeitsstaetten,AnzBesch,AnzBeschW,AnzBeschM,AnzVZA,AnzVZAW,AnzVZAM +2011,1,Ganze Stadt,0,Alle Rechtsformen,0,Alle Betriebsgrössen,39747,443621,202970,240651,339750,136406,203349 +2011,10,Kreis 1,0,Alle Rechtsformen,0,Alle Betriebsgrössen,5249,68627,34662,33965,53138,24338,28799 +2011,11,Rathaus,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1522,10180,5492,4688,7205,3493,3710 +2011,12,Hochschulen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,700,13203,7025,6178,9090,4376,4714 +2011,13,Lindenhof,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1193,15160,7649,7511,12213,5602,6608 +2011,14,City,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1834,30084,14496,15588,24633,10861,13768 +2011,20,Kreis 2,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3690,42295,19218,23077,32625,12786,19844 +2011,21,Wollishofen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1050,7578,3396,4182,5865,2271,3594 +2011,23,Leimbach,0,Alle Rechtsformen,0,Alle Betriebsgrössen,177,640,340,300,460,217,246 +2011,24,Enge,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2463,34077,15482,18595,26297,10298,16002 +2011,30,Kreis 3,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3716,41539,18494,23045,32146,12425,19724 +2011,31,Alt-Wiedikon,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1668,25737,10281,15456,20428,6962,13465 +2011,33,Friesenberg,0,Alle Rechtsformen,0,Alle Betriebsgrössen,358,4510,3113,1397,3523,2311,1209 +2011,34,Sihlfeld,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1690,11292,5100,6192,8196,3148,5049 +2011,40,Kreis 4,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3892,36061,15217,20844,27489,10031,17459 +2011,41,Werd,0,Alle Rechtsformen,0,Alle Betriebsgrössen,984,9023,4202,4821,6753,2834,3919 +2011,42,Langstrasse,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2066,19957,8639,11318,15238,5743,9495 +2011,44,Hard,0,Alle Rechtsformen,0,Alle Betriebsgrössen,842,7081,2376,4705,5497,1455,4045 +2011,50,Kreis 5,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2552,44156,16755,27401,35887,11991,23902 +2011,51,Gewerbeschule,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1363,15655,7201,8454,11596,4844,6753 +2011,52,Escher Wyss,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1189,28501,9554,18947,24293,7144,17146 +2011,60,Kreis 6,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2961,25736,12597,13139,18409,8037,10367 +2011,61,Unterstrass,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1931,15235,7380,7855,11198,4786,6413 +2011,63,Oberstrass,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1030,10501,5217,5284,7210,3253,3956 +2011,70,Kreis 7,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3758,31803,19099,12704,23623,13268,10356 +2011,71,Fluntern,0,Alle Rechtsformen,0,Alle Betriebsgrössen,838,14560,8961,5599,11196,6607,4590 +2011,72,Hottingen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1701,11377,6669,4708,8268,4448,3822 +2011,73,Hirslanden,0,Alle Rechtsformen,0,Alle Betriebsgrössen,715,3824,2212,1612,2678,1383,1296 +2011,74,Witikon,0,Alle Rechtsformen,0,Alle Betriebsgrössen,504,2042,1257,785,1483,833,647 +2011,80,Kreis 8,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3675,28912,15351,13561,21358,10330,11025 +2011,81,Seefeld,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1526,9963,4975,4988,7537,3393,4146 +2011,82,Mühlebach,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1503,9161,4556,4605,6891,3024,3869 +2011,83,Weinegg,0,Alle Rechtsformen,0,Alle Betriebsgrössen,646,9788,5820,3968,6925,3913,3011 +2011,90,Kreis 9,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3277,48818,18683,30135,38569,12314,26256 +2011,91,Albisrieden,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1024,7797,3310,4487,6040,2102,3940 +2011,92,Altstetten,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2253,41021,15373,25648,32530,10215,22315 +2011,100,Kreis 10,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1983,15272,7707,7565,10589,4753,5836 +2011,101,Höngg,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1031,8233,3643,4590,5796,2281,3515 +2011,102,Wipkingen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,952,7039,4064,2975,4795,2475,2322 +2011,110,Kreis 11,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3902,52782,21371,31411,40609,13940,26668 +2011,111,Affoltern,0,Alle Rechtsformen,0,Alle Betriebsgrössen,800,3553,1675,1878,2671,1084,1586 +2011,115,Oerlikon,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1767,23926,10162,13764,18381,6569,11812 +2011,119,Seebach,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1335,25303,9534,15769,19553,6288,13267 +2011,120,Kreis 12,0,Alle Rechtsformen,0,Alle Betriebsgrössen,945,6868,3415,3453,4762,1932,2828 +2011,121,Saatlen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,183,1516,493,1023,1197,319,879 +2011,122,Schwamendingen-Mitte,0,Alle Rechtsformen,0,Alle Betriebsgrössen,464,3690,2029,1661,2305,982,1325 +2011,123,Hirzenbach,0,Alle Rechtsformen,0,Alle Betriebsgrössen,298,1662,893,769,1258,636,624 +2011,990,Kreis Unbekannt,0,Alle Rechtsformen,0,Alle Betriebsgrössen,147,752,401,351,549,258,292 +2011,999,Quartier Unbekannt,0,Alle Rechtsformen,0,Alle Betriebsgrössen,147,752,401,351,549,258,292 +2012,1,Ganze Stadt,0,Alle Rechtsformen,0,Alle Betriebsgrössen,40362,448474,205879,242595,343562,138749,204816 diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_common_output.tmcf b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_common_output.tmcf deleted file mode 100644 index ac95a18697..0000000000 --- a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_common_output.tmcf +++ /dev/null @@ -1,9 +0,0 @@ -Node: E:wir_2552_wiki_output->E0 -observationAbout: C:wir_2552_wiki_output->observationAbout -observationDate: C:wir_2552_wiki_output->observationDate -value: C:wir_2552_wiki_output->value -variableMeasured: C:wir_2552_wiki_output->variableMeasured -unit: C:wir_2552_wiki_output->unit -scalingFactor: C:wir_2552_wiki_output->scalingFactor -measurementMethod: C:wir_2552_wiki_output->measurementMethod -typeOf: dcs:StatVarObservation diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.csv b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.csv index fa2612349d..3475920ecb 100644 --- a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.csv +++ b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.csv @@ -1,29 +1,337 @@ -observationAbout,observationDate,value,variableMeasured,unit,scalingFactor,measurementMethod -wikidataId/Q692511,2011,1522,dcid:Count_Company,,, -wikidataId/Q692511,2011,10180,dcid:Count_Person_Employed,,, -wikidataId/Q692511,2011,5492,dcid:Count_Person_Employed_Female,,, -wikidataId/Q692511,2011,4688,dcid:Count_Person_Employed_Male,,, -wikidataId/Q692511,2011,7205,dcid:Count_Person_FullTimeEmployee,,, -wikidataId/Q692511,2011,3493,dcid:Count_Person_FullTimeEmployee_Female,,, -wikidataId/Q692511,2011,3710,dcid:Count_Person_FullTimeEmployee_Male,,, -wikidataId/Q39240,2011,700,dcid:Count_Company,,, -wikidataId/Q39240,2011,13203,dcid:Count_Person_Employed,,, -wikidataId/Q39240,2011,7025,dcid:Count_Person_Employed_Female,,, -wikidataId/Q39240,2011,6178,dcid:Count_Person_Employed_Male,,, -wikidataId/Q39240,2011,9090,dcid:Count_Person_FullTimeEmployee,,, -wikidataId/Q39240,2011,4376,dcid:Count_Person_FullTimeEmployee_Female,,, -wikidataId/Q39240,2011,4714,dcid:Count_Person_FullTimeEmployee_Male,,, -wikidataId/Q10987378,2011,1193,dcid:Count_Company,,, -wikidataId/Q10987378,2011,15160,dcid:Count_Person_Employed,,, -wikidataId/Q10987378,2011,7649,dcid:Count_Person_Employed_Female,,, -wikidataId/Q10987378,2011,7511,dcid:Count_Person_Employed_Male,,, -wikidataId/Q10987378,2011,12213,dcid:Count_Person_FullTimeEmployee,,, -wikidataId/Q10987378,2011,5602,dcid:Count_Person_FullTimeEmployee_Female,,, -wikidataId/Q10987378,2011,6608,dcid:Count_Person_FullTimeEmployee_Male,,, -wikidataId/Q1093831,2011,1834,dcid:Count_Company,,, -wikidataId/Q1093831,2011,30084,dcid:Count_Person_Employed,,, -wikidataId/Q1093831,2011,14496,dcid:Count_Person_Employed_Female,,, -wikidataId/Q1093831,2011,15588,dcid:Count_Person_Employed_Male,,, -wikidataId/Q1093831,2011,24633,dcid:Count_Person_FullTimeEmployee,,, -wikidataId/Q1093831,2011,10861,dcid:Count_Person_FullTimeEmployee_Female,,, -wikidataId/Q1093831,2011,13768,dcid:Count_Person_FullTimeEmployee_Male,,, +observationAbout,observationDate,value,variableMeasured,unit +wikidataId/Q72,2011,39747,dcid:Count_Company, +wikidataId/Q72,2011,443621,dcid:Count_Person_Employed, +wikidataId/Q72,2011,202970,dcid:Count_Person_Employed_Female, +wikidataId/Q72,2011,240651,dcid:Count_Person_Employed_Male, +wikidataId/Q72,2011,339750,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q72,2011,136406,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q72,2011,203349,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q445559,2011,5249,dcid:Count_Company, +wikidataId/Q445559,2011,68627,dcid:Count_Person_Employed, +wikidataId/Q445559,2011,34662,dcid:Count_Person_Employed_Female, +wikidataId/Q445559,2011,33965,dcid:Count_Person_Employed_Male, +wikidataId/Q445559,2011,53138,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q445559,2011,24338,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q445559,2011,28799,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q692511,2011,1522,dcid:Count_Company, +wikidataId/Q692511,2011,10180,dcid:Count_Person_Employed, +wikidataId/Q692511,2011,5492,dcid:Count_Person_Employed_Female, +wikidataId/Q692511,2011,4688,dcid:Count_Person_Employed_Male, +wikidataId/Q692511,2011,7205,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q692511,2011,3493,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q692511,2011,3710,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q39240,2011,700,dcid:Count_Company, +wikidataId/Q39240,2011,13203,dcid:Count_Person_Employed, +wikidataId/Q39240,2011,7025,dcid:Count_Person_Employed_Female, +wikidataId/Q39240,2011,6178,dcid:Count_Person_Employed_Male, +wikidataId/Q39240,2011,9090,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q39240,2011,4376,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q39240,2011,4714,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q10987378,2011,1193,dcid:Count_Company, +wikidataId/Q10987378,2011,15160,dcid:Count_Person_Employed, +wikidataId/Q10987378,2011,7649,dcid:Count_Person_Employed_Female, +wikidataId/Q10987378,2011,7511,dcid:Count_Person_Employed_Male, +wikidataId/Q10987378,2011,12213,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q10987378,2011,5602,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q10987378,2011,6608,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q1093831,2011,1834,dcid:Count_Company, +wikidataId/Q1093831,2011,30084,dcid:Count_Person_Employed, +wikidataId/Q1093831,2011,14496,dcid:Count_Person_Employed_Female, +wikidataId/Q1093831,2011,15588,dcid:Count_Person_Employed_Male, +wikidataId/Q1093831,2011,24633,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q1093831,2011,10861,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q1093831,2011,13768,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q456153,2011,3690,dcid:Count_Company, +wikidataId/Q456153,2011,42295,dcid:Count_Person_Employed, +wikidataId/Q456153,2011,19218,dcid:Count_Person_Employed_Female, +wikidataId/Q456153,2011,23077,dcid:Count_Person_Employed_Male, +wikidataId/Q456153,2011,32625,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q456153,2011,12786,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q456153,2011,19844,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q642353,2011,1050,dcid:Count_Company, +wikidataId/Q642353,2011,7578,dcid:Count_Person_Employed, +wikidataId/Q642353,2011,3396,dcid:Count_Person_Employed_Female, +wikidataId/Q642353,2011,4182,dcid:Count_Person_Employed_Male, +wikidataId/Q642353,2011,5865,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q642353,2011,2271,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q642353,2011,3594,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q691367,2011,177,dcid:Count_Company, +wikidataId/Q691367,2011,640,dcid:Count_Person_Employed, +wikidataId/Q691367,2011,340,dcid:Count_Person_Employed_Female, +wikidataId/Q691367,2011,300,dcid:Count_Person_Employed_Male, +wikidataId/Q691367,2011,460,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q691367,2011,217,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q691367,2011,246,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q648218,2011,2463,dcid:Count_Company, +wikidataId/Q648218,2011,34077,dcid:Count_Person_Employed, +wikidataId/Q648218,2011,15482,dcid:Count_Person_Employed_Female, +wikidataId/Q648218,2011,18595,dcid:Count_Person_Employed_Male, +wikidataId/Q648218,2011,26297,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q648218,2011,10298,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q648218,2011,16002,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q675017,2011,3716,dcid:Count_Company, +wikidataId/Q675017,2011,41539,dcid:Count_Person_Employed, +wikidataId/Q675017,2011,18494,dcid:Count_Person_Employed_Female, +wikidataId/Q675017,2011,23045,dcid:Count_Person_Employed_Male, +wikidataId/Q675017,2011,32146,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q675017,2011,12425,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q675017,2011,19724,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q433012,2011,1668,dcid:Count_Company, +wikidataId/Q433012,2011,25737,dcid:Count_Person_Employed, +wikidataId/Q433012,2011,10281,dcid:Count_Person_Employed_Female, +wikidataId/Q433012,2011,15456,dcid:Count_Person_Employed_Male, +wikidataId/Q433012,2011,20428,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q433012,2011,6962,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q433012,2011,13465,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q693357,2011,358,dcid:Count_Company, +wikidataId/Q693357,2011,4510,dcid:Count_Person_Employed, +wikidataId/Q693357,2011,3113,dcid:Count_Person_Employed_Female, +wikidataId/Q693357,2011,1397,dcid:Count_Person_Employed_Male, +wikidataId/Q693357,2011,3523,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q693357,2011,2311,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q693357,2011,1209,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q370104,2011,1690,dcid:Count_Company, +wikidataId/Q370104,2011,11292,dcid:Count_Person_Employed, +wikidataId/Q370104,2011,5100,dcid:Count_Person_Employed_Female, +wikidataId/Q370104,2011,6192,dcid:Count_Person_Employed_Male, +wikidataId/Q370104,2011,8196,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q370104,2011,3148,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q370104,2011,5049,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q677133,2011,3892,dcid:Count_Company, +wikidataId/Q677133,2011,36061,dcid:Count_Person_Employed, +wikidataId/Q677133,2011,15217,dcid:Count_Person_Employed_Female, +wikidataId/Q677133,2011,20844,dcid:Count_Person_Employed_Male, +wikidataId/Q677133,2011,27489,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q677133,2011,10031,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q677133,2011,17459,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q531899,2011,984,dcid:Count_Company, +wikidataId/Q531899,2011,9023,dcid:Count_Person_Employed, +wikidataId/Q531899,2011,4202,dcid:Count_Person_Employed_Female, +wikidataId/Q531899,2011,4821,dcid:Count_Person_Employed_Male, +wikidataId/Q531899,2011,6753,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q531899,2011,2834,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q531899,2011,3919,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q1805410,2011,2066,dcid:Count_Company, +wikidataId/Q1805410,2011,19957,dcid:Count_Person_Employed, +wikidataId/Q1805410,2011,8639,dcid:Count_Person_Employed_Female, +wikidataId/Q1805410,2011,11318,dcid:Count_Person_Employed_Male, +wikidataId/Q1805410,2011,15238,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q1805410,2011,5743,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q1805410,2011,9495,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q870084,2011,842,dcid:Count_Company, +wikidataId/Q870084,2011,7081,dcid:Count_Person_Employed, +wikidataId/Q870084,2011,2376,dcid:Count_Person_Employed_Female, +wikidataId/Q870084,2011,4705,dcid:Count_Person_Employed_Male, +wikidataId/Q870084,2011,5497,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q870084,2011,1455,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q870084,2011,4045,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q460885,2011,2552,dcid:Count_Company, +wikidataId/Q460885,2011,44156,dcid:Count_Person_Employed, +wikidataId/Q460885,2011,16755,dcid:Count_Person_Employed_Female, +wikidataId/Q460885,2011,27401,dcid:Count_Person_Employed_Male, +wikidataId/Q460885,2011,35887,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q460885,2011,11991,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q460885,2011,23902,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q693413,2011,1363,dcid:Count_Company, +wikidataId/Q693413,2011,15655,dcid:Count_Person_Employed, +wikidataId/Q693413,2011,7201,dcid:Count_Person_Employed_Female, +wikidataId/Q693413,2011,8454,dcid:Count_Person_Employed_Male, +wikidataId/Q693413,2011,11596,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q693413,2011,4844,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q693413,2011,6753,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q687052,2011,1189,dcid:Count_Company, +wikidataId/Q687052,2011,28501,dcid:Count_Person_Employed, +wikidataId/Q687052,2011,9554,dcid:Count_Person_Employed_Female, +wikidataId/Q687052,2011,18947,dcid:Count_Person_Employed_Male, +wikidataId/Q687052,2011,24293,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q687052,2011,7144,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q687052,2011,17146,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q456170,2011,2961,dcid:Count_Company, +wikidataId/Q456170,2011,25736,dcid:Count_Person_Employed, +wikidataId/Q456170,2011,12597,dcid:Count_Person_Employed_Female, +wikidataId/Q456170,2011,13139,dcid:Count_Person_Employed_Male, +wikidataId/Q456170,2011,18409,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q456170,2011,8037,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q456170,2011,10367,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q656446,2011,1931,dcid:Count_Company, +wikidataId/Q656446,2011,15235,dcid:Count_Person_Employed, +wikidataId/Q656446,2011,7380,dcid:Count_Person_Employed_Female, +wikidataId/Q656446,2011,7855,dcid:Count_Person_Employed_Male, +wikidataId/Q656446,2011,11198,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q656446,2011,4786,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q656446,2011,6413,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q693483,2011,1030,dcid:Count_Company, +wikidataId/Q693483,2011,10501,dcid:Count_Person_Employed, +wikidataId/Q693483,2011,5217,dcid:Count_Person_Employed_Female, +wikidataId/Q693483,2011,5284,dcid:Count_Person_Employed_Male, +wikidataId/Q693483,2011,7210,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q693483,2011,3253,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q693483,2011,3956,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q456149,2011,3758,dcid:Count_Company, +wikidataId/Q456149,2011,31803,dcid:Count_Person_Employed, +wikidataId/Q456149,2011,19099,dcid:Count_Person_Employed_Female, +wikidataId/Q456149,2011,12704,dcid:Count_Person_Employed_Male, +wikidataId/Q456149,2011,23623,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q456149,2011,13268,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q456149,2011,10356,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q693269,2011,838,dcid:Count_Company, +wikidataId/Q693269,2011,14560,dcid:Count_Person_Employed, +wikidataId/Q693269,2011,8961,dcid:Count_Person_Employed_Female, +wikidataId/Q693269,2011,5599,dcid:Count_Person_Employed_Male, +wikidataId/Q693269,2011,11196,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q693269,2011,6607,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q693269,2011,4590,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q693454,2011,1701,dcid:Count_Company, +wikidataId/Q693454,2011,11377,dcid:Count_Person_Employed, +wikidataId/Q693454,2011,6669,dcid:Count_Person_Employed_Female, +wikidataId/Q693454,2011,4708,dcid:Count_Person_Employed_Male, +wikidataId/Q693454,2011,8268,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q693454,2011,4448,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q693454,2011,3822,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q476940,2011,715,dcid:Count_Company, +wikidataId/Q476940,2011,3824,dcid:Count_Person_Employed, +wikidataId/Q476940,2011,2212,dcid:Count_Person_Employed_Female, +wikidataId/Q476940,2011,1612,dcid:Count_Person_Employed_Male, +wikidataId/Q476940,2011,2678,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q476940,2011,1383,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q476940,2011,1296,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q392079,2011,504,dcid:Count_Company, +wikidataId/Q392079,2011,2042,dcid:Count_Person_Employed, +wikidataId/Q392079,2011,1257,dcid:Count_Person_Employed_Female, +wikidataId/Q392079,2011,785,dcid:Count_Person_Employed_Male, +wikidataId/Q392079,2011,1483,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q392079,2011,833,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q392079,2011,647,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q677710,2011,3675,dcid:Count_Company, +wikidataId/Q677710,2011,28912,dcid:Count_Person_Employed, +wikidataId/Q677710,2011,15351,dcid:Count_Person_Employed_Female, +wikidataId/Q677710,2011,13561,dcid:Count_Person_Employed_Male, +wikidataId/Q677710,2011,21358,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q677710,2011,10330,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q677710,2011,11025,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q692773,2011,1526,dcid:Count_Company, +wikidataId/Q692773,2011,9963,dcid:Count_Person_Employed, +wikidataId/Q692773,2011,4975,dcid:Count_Person_Employed_Female, +wikidataId/Q692773,2011,4988,dcid:Count_Person_Employed_Male, +wikidataId/Q692773,2011,7537,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q692773,2011,3393,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q692773,2011,4146,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q693397,2011,1503,dcid:Count_Company, +wikidataId/Q693397,2011,9161,dcid:Count_Person_Employed, +wikidataId/Q693397,2011,4556,dcid:Count_Person_Employed_Female, +wikidataId/Q693397,2011,4605,dcid:Count_Person_Employed_Male, +wikidataId/Q693397,2011,6891,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q693397,2011,3024,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q693397,2011,3869,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q693321,2011,646,dcid:Count_Company, +wikidataId/Q693321,2011,9788,dcid:Count_Person_Employed, +wikidataId/Q693321,2011,5820,dcid:Count_Person_Employed_Female, +wikidataId/Q693321,2011,3968,dcid:Count_Person_Employed_Male, +wikidataId/Q693321,2011,6925,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q693321,2011,3913,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q693321,2011,3011,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q456159,2011,3277,dcid:Count_Company, +wikidataId/Q456159,2011,48818,dcid:Count_Person_Employed, +wikidataId/Q456159,2011,18683,dcid:Count_Person_Employed_Female, +wikidataId/Q456159,2011,30135,dcid:Count_Person_Employed_Male, +wikidataId/Q456159,2011,38569,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q456159,2011,12314,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q456159,2011,26256,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q80797,2011,1024,dcid:Count_Company, +wikidataId/Q80797,2011,7797,dcid:Count_Person_Employed, +wikidataId/Q80797,2011,3310,dcid:Count_Person_Employed_Female, +wikidataId/Q80797,2011,4487,dcid:Count_Person_Employed_Male, +wikidataId/Q80797,2011,6040,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q80797,2011,2102,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q80797,2011,3940,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q445711,2011,2253,dcid:Count_Company, +wikidataId/Q445711,2011,41021,dcid:Count_Person_Employed, +wikidataId/Q445711,2011,15373,dcid:Count_Person_Employed_Female, +wikidataId/Q445711,2011,25648,dcid:Count_Person_Employed_Male, +wikidataId/Q445711,2011,32530,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q445711,2011,10215,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q445711,2011,22315,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q456175,2011,1983,dcid:Count_Company, +wikidataId/Q456175,2011,15272,dcid:Count_Person_Employed, +wikidataId/Q456175,2011,7707,dcid:Count_Person_Employed_Female, +wikidataId/Q456175,2011,7565,dcid:Count_Person_Employed_Male, +wikidataId/Q456175,2011,10589,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q456175,2011,4753,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q456175,2011,5836,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q455496,2011,1031,dcid:Count_Company, +wikidataId/Q455496,2011,8233,dcid:Count_Person_Employed, +wikidataId/Q455496,2011,3643,dcid:Count_Person_Employed_Female, +wikidataId/Q455496,2011,4590,dcid:Count_Person_Employed_Male, +wikidataId/Q455496,2011,5796,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q455496,2011,2281,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q455496,2011,3515,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q678030,2011,952,dcid:Count_Company, +wikidataId/Q678030,2011,7039,dcid:Count_Person_Employed, +wikidataId/Q678030,2011,4064,dcid:Count_Person_Employed_Female, +wikidataId/Q678030,2011,2975,dcid:Count_Person_Employed_Male, +wikidataId/Q678030,2011,4795,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q678030,2011,2475,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q678030,2011,2322,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q870098,2011,3902,dcid:Count_Company, +wikidataId/Q870098,2011,52782,dcid:Count_Person_Employed, +wikidataId/Q870098,2011,21371,dcid:Count_Person_Employed_Female, +wikidataId/Q870098,2011,31411,dcid:Count_Person_Employed_Male, +wikidataId/Q870098,2011,40609,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q870098,2011,13940,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q870098,2011,26668,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q382903,2011,800,dcid:Count_Company, +wikidataId/Q382903,2011,3553,dcid:Count_Person_Employed, +wikidataId/Q382903,2011,1675,dcid:Count_Person_Employed_Female, +wikidataId/Q382903,2011,1878,dcid:Count_Person_Employed_Male, +wikidataId/Q382903,2011,2671,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q382903,2011,1084,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q382903,2011,1586,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q167179,2011,1767,dcid:Count_Company, +wikidataId/Q167179,2011,23926,dcid:Count_Person_Employed, +wikidataId/Q167179,2011,10162,dcid:Count_Person_Employed_Female, +wikidataId/Q167179,2011,13764,dcid:Count_Person_Employed_Male, +wikidataId/Q167179,2011,18381,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q167179,2011,6569,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q167179,2011,11812,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q276792,2011,1335,dcid:Count_Company, +wikidataId/Q276792,2011,25303,dcid:Count_Person_Employed, +wikidataId/Q276792,2011,9534,dcid:Count_Person_Employed_Female, +wikidataId/Q276792,2011,15769,dcid:Count_Person_Employed_Male, +wikidataId/Q276792,2011,19553,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q276792,2011,6288,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q276792,2011,13267,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q657525,2011,945,dcid:Count_Company, +wikidataId/Q657525,2011,6868,dcid:Count_Person_Employed, +wikidataId/Q657525,2011,3415,dcid:Count_Person_Employed_Female, +wikidataId/Q657525,2011,3453,dcid:Count_Person_Employed_Male, +wikidataId/Q657525,2011,4762,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q657525,2011,1932,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q657525,2011,2828,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q652455,2011,183,dcid:Count_Company, +wikidataId/Q652455,2011,1516,dcid:Count_Person_Employed, +wikidataId/Q652455,2011,493,dcid:Count_Person_Employed_Female, +wikidataId/Q652455,2011,1023,dcid:Count_Person_Employed_Male, +wikidataId/Q652455,2011,1197,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q652455,2011,319,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q652455,2011,879,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q692728,2011,464,dcid:Count_Company, +wikidataId/Q692728,2011,3690,dcid:Count_Person_Employed, +wikidataId/Q692728,2011,2029,dcid:Count_Person_Employed_Female, +wikidataId/Q692728,2011,1661,dcid:Count_Person_Employed_Male, +wikidataId/Q692728,2011,2305,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q692728,2011,982,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q692728,2011,1325,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q693374,2011,298,dcid:Count_Company, +wikidataId/Q693374,2011,1662,dcid:Count_Person_Employed, +wikidataId/Q693374,2011,893,dcid:Count_Person_Employed_Female, +wikidataId/Q693374,2011,769,dcid:Count_Person_Employed_Male, +wikidataId/Q693374,2011,1258,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q693374,2011,636,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q693374,2011,624,dcid:Count_Person_FullTimeEmployee_Male, +wikidataId/Q72,2012,40362,dcid:Count_Company, +wikidataId/Q72,2012,448474,dcid:Count_Person_Employed, +wikidataId/Q72,2012,205879,dcid:Count_Person_Employed_Female, +wikidataId/Q72,2012,242595,dcid:Count_Person_Employed_Male, +wikidataId/Q72,2012,343562,dcid:Count_Person_FullTimeEmployee, +wikidataId/Q72,2012,138749,dcid:Count_Person_FullTimeEmployee_Female, +wikidataId/Q72,2012,204816,dcid:Count_Person_FullTimeEmployee_Male, diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.tmcf b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.tmcf new file mode 100644 index 0000000000..9b9c3598f7 --- /dev/null +++ b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.tmcf @@ -0,0 +1,7 @@ +Node: E:zurich_population_wir_2552_wiki_output->E0 +observationAbout: C:zurich_population_wir_2552_wiki_output->observationAbout +observationDate: C:zurich_population_wir_2552_wiki_output->observationDate +value: C:zurich_population_wir_2552_wiki_output->value +variableMeasured: C:zurich_population_wir_2552_wiki_output->variableMeasured +unit: C:zurich_population_wir_2552_wiki_output->unit +typeOf: dcs:StatVarObservation diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output_stat_vars.mcf b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output_stat_vars.mcf new file mode 100644 index 0000000000..818487edb0 --- /dev/null +++ b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output_stat_vars.mcf @@ -0,0 +1,25 @@ +# Auto generated using command: "tools/statvar_importer/stat_var_processor.py --input_data=statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv --pv_map=statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv --config_file=statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv --output_path=statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output" on 2026-08-05 11:36:15.561091 + +Node: dcid:Count_Person_Employed +typeOf: dcid:StatisticalVariable +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +employmentStatus: dcid:Employed + +Node: dcid:Count_Person_Employed_Female +typeOf: dcid:StatisticalVariable +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +employmentStatus: dcid:Employed +gender: dcid:Female + +Node: dcid:Count_Person_Employed_Male +typeOf: dcid:StatisticalVariable +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +employmentStatus: dcid:Employed +gender: dcid:Male + diff --git a/statvar_imports/zurich/wir_2552_wiki/validation_config.json b/statvar_imports/zurich/wir_2552_wiki/validation_config.json new file mode 100644 index 0000000000..66014b3369 --- /dev/null +++ b/statvar_imports/zurich/wir_2552_wiki/validation_config.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + } + ] +} diff --git a/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv b/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv index f554cae90e..f70ab81c09 100644 --- a/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv +++ b/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv @@ -1,5 +1,5 @@ parameter,value -url,https://www.stadt-zuerich.ch/portal/de/index.html +url,https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552 description, place_type,Country #places_within, @@ -8,7 +8,8 @@ end_date, release_frequency, process, comments, -output_columns,"observationAbout,observationDate,value,variableMeasured,unit,scalingFactor,measurementMethod" +output_columns,"observationAbout,observationDate,value,variableMeasured" header_rows,1 -#input_rows,10 -#Import_Name,Zurich_Population_Number_Of_Company_Workplace_Employees +#aggregate_duplicate_svobs,sum +reconcile_nodes,False +output_counters,wir_2552_wiki/counters/zurich_population_wir_2552_wiki_counters.csv diff --git a/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv b/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv index db78483403..f609671e63 100644 --- a/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv +++ b/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv @@ -1,10 +1,56 @@ -key,prop,val,p1,v1,,,, -Zeit,observationDate,{Number},,,,,, -employees,value,{Number},populationType,Person,employmentStatus,Employed,, -companies,value,{Number},populationType,Company,,,, -employees_f,value,{Number},populationType,Person,employmentStatus,Employed,gender,Female -employees_m,value,{Number},populationType,Person,employmentStatus,Employed,gender,Male -fte,value,{Number},populationType,Person,employmentStatus,FullTimeEmployee,, -fte_f,value,{Number},populationType,Person,employmentStatus,FullTimeEmployee,gender,Female -fte_m,value,{Number},populationType,Person,employmentStatus,FullTimeEmployee,gender,Male -wikiId,observationAbout,wikidataId/{Data},,,,,, \ No newline at end of file +key,prop,val,p1,v1,,,, +Jahr,observationDate,{Number},,,,,, +AnzBesch,value,{Number},populationType,Person,employmentStatus,Employed,, +Arbeitsstaetten,value,{Number},populationType,Company,,,, +AnzBeschW,value,{Number},populationType,Person,employmentStatus,Employed,gender,Female +AnzBeschM,value,{Number},populationType,Person,employmentStatus,Employed,gender,Male +AnzVZA,value,{Number},populationType,Person,employmentStatus,FullTimeEmployee,, +AnzVZAW,value,{Number},populationType,Person,employmentStatus,FullTimeEmployee,gender,Female +AnzVZAM,value,{Number},populationType,Person,employmentStatus,FullTimeEmployee,gender,Male +RaumLang:Ganze Stadt,observationAbout,wikidataId/Q72,,,,,, +RaumLang:Kreis 1,observationAbout,wikidataId/Q445559,,,,,, +RaumLang:Kreis 2,observationAbout,wikidataId/Q456153,,,,,, +RaumLang:Kreis 3,observationAbout,wikidataId/Q675017,,,,,, +RaumLang:Kreis 4,observationAbout,wikidataId/Q677133,,,,,, +RaumLang:Kreis 5,observationAbout,wikidataId/Q460885,,,,,, +RaumLang:Kreis 6,observationAbout,wikidataId/Q456170,,,,,, +RaumLang:Kreis 7,observationAbout,wikidataId/Q456149,,,,,, +RaumLang:Kreis 8,observationAbout,wikidataId/Q677710,,,,,, +RaumLang:Kreis 9,observationAbout,wikidataId/Q456159,,,,,, +RaumLang:Kreis 10,observationAbout,wikidataId/Q456175,,,,,, +RaumLang:Kreis 11,observationAbout,wikidataId/Q870098,,,,,, +RaumLang:Kreis 12,observationAbout,wikidataId/Q657525,,,,,, +RaumLang:Rathaus,observationAbout,wikidataId/Q692511,,,,,, +RaumLang:Hochschulen,observationAbout,wikidataId/Q39240,,,,,, +RaumLang:Lindenhof,observationAbout,wikidataId/Q10987378,,,,,, +RaumLang:City,observationAbout,wikidataId/Q1093831,,,,,, +RaumLang:Wollishofen,observationAbout,wikidataId/Q642353,,,,,, +RaumLang:Leimbach,observationAbout,wikidataId/Q691367,,,,,, +RaumLang:Enge,observationAbout,wikidataId/Q648218,,,,,, +RaumLang:Alt-Wiedikon,observationAbout,wikidataId/Q433012,,,,,, +RaumLang:Friesenberg,observationAbout,wikidataId/Q693357,,,,,, +RaumLang:Sihlfeld,observationAbout,wikidataId/Q370104,,,,,, +RaumLang:Werd,observationAbout,wikidataId/Q531899,,,,,, +RaumLang:Langstrasse,observationAbout,wikidataId/Q1805410,,,,,, +RaumLang:Hard,observationAbout,wikidataId/Q870084,,,,,, +RaumLang:Gewerbeschule,observationAbout,wikidataId/Q693413,,,,,, +RaumLang:Escher Wyss,observationAbout,wikidataId/Q687052,,,,,, +RaumLang:Unterstrass,observationAbout,wikidataId/Q656446,,,,,, +RaumLang:Oberstrass,observationAbout,wikidataId/Q693483,,,,,, +RaumLang:Fluntern,observationAbout,wikidataId/Q693269,,,,,, +RaumLang:Hottingen,observationAbout,wikidataId/Q693454,,,,,, +RaumLang:Hirslanden,observationAbout,wikidataId/Q476940,,,,,, +RaumLang:Witikon,observationAbout,wikidataId/Q392079,,,,,, +RaumLang:Seefeld,observationAbout,wikidataId/Q692773,,,,,, +RaumLang:Mühlebach,observationAbout,wikidataId/Q693397,,,,,, +RaumLang:Weinegg,observationAbout,wikidataId/Q693321,,,,,, +RaumLang:Albisrieden,observationAbout,wikidataId/Q80797,,,,,, +RaumLang:Altstetten,observationAbout,wikidataId/Q445711,,,,,, +RaumLang:Höngg,observationAbout,wikidataId/Q455496,,,,,, +RaumLang:Wipkingen,observationAbout,wikidataId/Q678030,,,,,, +RaumLang:Affoltern,observationAbout,wikidataId/Q382903,,,,,, +RaumLang:Oerlikon,observationAbout,wikidataId/Q167179,,,,,, +RaumLang:Seebach,observationAbout,wikidataId/Q276792,,,,,, +RaumLang:Saatlen,observationAbout,wikidataId/Q652455,,,,,, +RaumLang:Schwamendingen-Mitte,observationAbout,wikidataId/Q692728,,,,,, +RaumLang:Hirzenbach,observationAbout,wikidataId/Q693374,,,,,, \ No newline at end of file From cbbe5a0435887c500d00c73b21284882725f857d Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Thu, 3 Sep 2026 10:54:59 +0000 Subject: [PATCH 2/6] Address code review feedback on Zurich imports - Correct place_type from Country to City in metadata configs - Reduce resource_limits to cpu: 1, memory: 2, disk: 10 in manifest.json - Add defensive validation checks to generate_rollups.py for empty data and missing columns TAG=agy CONV=e14d0b1e-8efd-44a0-b3d1-5640746680a4 --- .../bev_3240_wiki/bev_3240_wiki_metadata.csv | 2 +- statvar_imports/zurich/manifest.json | 12 ++++++------ .../zurich/wir_2552_wiki/generate_rollups.py | 17 +++++++++++++++++ .../wir_2552_wiki/wir_2552_wiki_metadata.csv | 2 +- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv b/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv index 3db1541727..9bdbc3d1b1 100644 --- a/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv +++ b/statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv @@ -1,7 +1,7 @@ parameter,value url,https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240 description, -place_type,Country +place_type,City #places_within, start_date, end_date, diff --git a/statvar_imports/zurich/manifest.json b/statvar_imports/zurich/manifest.json index 0ca7ee230f..a0e30b8434 100644 --- a/statvar_imports/zurich/manifest.json +++ b/statvar_imports/zurich/manifest.json @@ -24,9 +24,9 @@ } ], "resource_limits": { - "cpu": 8, - "memory": 64, - "disk": 100 + "cpu": 1, + "memory": 2, + "disk": 10 }, "cron_schedule": "0 2 29 * *", "validation_config_file": "wir_2552_wiki/validation_config.json" @@ -75,9 +75,9 @@ } ], "resource_limits": { - "cpu": 8, - "memory": 64, - "disk": 100 + "cpu": 1, + "memory": 2, + "disk": 10 }, "cron_schedule": "30 11 29 * *", "validation_config_file": "bev_3240_wiki/validation_config.json" diff --git a/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py b/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py index 402dd933c2..452fb21b13 100644 --- a/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py +++ b/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py @@ -17,15 +17,32 @@ def main(): except UnicodeDecodeError: df = pd.read_csv(INPUT_RAW, encoding='iso-8859-1') + if df.empty: + raise ValueError(f"Input file {INPUT_RAW} is empty.") + + required_filter_cols = ['RechtsformSort', 'BetriebsgrSort'] + missing_cols = [col for col in required_filter_cols if col not in df.columns] + if missing_cols: + raise KeyError( + f"Input file {INPUT_RAW} is missing required filter columns: {missing_cols}. " + f"Available columns: {list(df.columns)}" + ) + # Keep only the total rows (Alle Rechtsformen and Alle Betriebsgrössen) df_rollups = df[(df['RechtsformSort'] == 0) & (df['BetriebsgrSort'] == 0)].copy() + if df_rollups.empty: + raise ValueError( + f"No rows matched filter (RechtsformSort == 0 & BetriebsgrSort == 0) in {INPUT_RAW}." + ) + # Convert numeric columns, converting non-numeric markers (e.g. 'K') to NaN/numeric val_cols = ['Arbeitsstaetten', 'AnzBesch', 'AnzBeschW', 'AnzBeschM', 'AnzVZA', 'AnzVZAW', 'AnzVZAM'] for col in val_cols: if col in df_rollups.columns: df_rollups[col] = pd.to_numeric(df_rollups[col], errors='coerce') + os.makedirs(os.path.dirname(OUTPUT_ROLLUPS), exist_ok=True) df_rollups.to_csv(OUTPUT_ROLLUPS, index=False, encoding='utf-8') print(f"Successfully generated {OUTPUT_ROLLUPS} with {len(df_rollups)} rows.") diff --git a/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv b/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv index f70ab81c09..515a509b02 100644 --- a/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv +++ b/statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv @@ -1,7 +1,7 @@ parameter,value url,https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552 description, -place_type,Country +place_type,City #places_within, start_date, end_date, From 8a027f3f0e083f5e3245df72a2ed33604ae16a2c Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Thu, 3 Sep 2026 12:23:29 +0000 Subject: [PATCH 3/6] Tune disk resource limit to 30GB to prevent OS disk exhaustion - Cloud Batch requires sufficient boot disk space for Debian 12 GCE base image + container runtime + python virtual environments. A 10GB disk encounters 'no space left on device'. - Configured disk to 30GB with cpu: 1 and memory: 2, verified working in dev. TAG=agy CONV=e14d0b1e-8efd-44a0-b3d1-5640746680a4 --- statvar_imports/zurich/manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/statvar_imports/zurich/manifest.json b/statvar_imports/zurich/manifest.json index a0e30b8434..a788a25cb9 100644 --- a/statvar_imports/zurich/manifest.json +++ b/statvar_imports/zurich/manifest.json @@ -26,7 +26,7 @@ "resource_limits": { "cpu": 1, "memory": 2, - "disk": 10 + "disk": 30 }, "cron_schedule": "0 2 29 * *", "validation_config_file": "wir_2552_wiki/validation_config.json" @@ -77,7 +77,7 @@ "resource_limits": { "cpu": 1, "memory": 2, - "disk": 10 + "disk": 30 }, "cron_schedule": "30 11 29 * *", "validation_config_file": "bev_3240_wiki/validation_config.json" From 23e7eb844f5ee434b305542f4bb908fb8fcaada0 Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Thu, 3 Sep 2026 13:36:43 +0000 Subject: [PATCH 4/6] Address adversarial code review audit findings for Zurich imports - Validation Configs: Added MAX_DATE_CONSISTENT and SQL_VALIDATOR date freshness rules for both imports. - Preprocessor: Hardened generate_rollups.py with absl flags, empty-file checks, and numeric coercion. - Unit Tests: Added comprehensive hermetic test suite generate_rollups_test.py (9/9 passing). - Manifest: Included raw WIR255OD2552.csv in source_files and adopted node_mcf wildcard pattern. - Test Fixtures: Removed unused trailing unit column in test CSV and TMCF to match 4-column runtime schema. - Documentation: Updated README.md geographic level descriptions. BUG=542856409,542859819 TAG=agy CONV=e14d0b1e-8efd-44a0-b3d1-5640746680a4 --- .gitignore | 3 + statvar_imports/zurich/README.md | 16 +- ...zurich_population_bev_3240_wiki_output.csv | 102 +-- ...urich_population_bev_3240_wiki_output.tmcf | 1 - .../bev_3240_wiki/validation_config.json | 15 + statvar_imports/zurich/manifest.json | 6 +- .../zurich/wir_2552_wiki/generate_rollups.py | 132 +++- .../wir_2552_wiki/generate_rollups_test.py | 157 ++++ .../test_data/wir_2552_wiki_raw_input.csv | 57 ++ ...zurich_population_wir_2552_wiki_output.csv | 674 +++++++++--------- ...urich_population_wir_2552_wiki_output.tmcf | 1 - ...ulation_wir_2552_wiki_output_stat_vars.mcf | 35 +- .../wir_2552_wiki/validation_config.json | 15 + 13 files changed, 788 insertions(+), 426 deletions(-) create mode 100644 statvar_imports/zurich/wir_2552_wiki/generate_rollups_test.py create mode 100644 statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_raw_input.csv diff --git a/.gitignore b/.gitignore index 87f3d76b24..f898268d3c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,7 @@ import-automation/executor/config_override.json # Ignore generated troubleshooting post-mortem documents agents/troubleshooting/ +# Local past conversation shortcuts +past_conversations/ + diff --git a/statvar_imports/zurich/README.md b/statvar_imports/zurich/README.md index 8983f1ee04..81a81cd067 100644 --- a/statvar_imports/zurich/README.md +++ b/statvar_imports/zurich/README.md @@ -1,7 +1,7 @@ 1. import_name": "Zurich_Population_Number_Of_Company_Workplace_Employees" 2. Import Overview -Number of companies, workplaces and employees in Zurich city at Province and City Level. +Number of companies, workplaces and employees in Zurich city at City, District (Kreise), and Quarter (Quartiere) Level. Source URL: [BFS WIR STATENT Data](https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552) Import Type: Fully Autorefresh Source Data Availability: 2011 to 2022 @@ -25,7 +25,7 @@ Fully Autorefresh:"0 2 29 * * " (Runs at 2:00 AM on the 29th day of every month) 1. import_name": "Zurich_Population_By_Age" 2. Import Overview -Zurich population data at Province and City Level. +Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level. Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3903_age10_wiki.csv) Import Type: Fully Autorefresh Source Data Availability: 1993 to 2023 @@ -47,7 +47,7 @@ Fully Autorefresh:" 0 6 29 * * " (Runs at 6:00 AM on the 29th of every month). 1. import_name": "Zurich_Population" 2. Import Overview -Total population of Zurich city by quarter and year at Province and City Level. +Total population of Zurich city by quarter and year at City, District (Kreise), and Quarter (Quartiere) Level. Source URL: [BEV324OD3240 Dataset](https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240) Import Type: Fully Autorefresh Source Data Availability: 1941 to 2023 @@ -69,7 +69,7 @@ Fully Autorefresh:" 30 11 29 * * " (Runs at 11:30 AM on the 29th of every month) 1. import_name": "Zurich_Population_Number_Of_Birth_By_Origin" 2. Import Overview -Zurich population data at Province and City Level. +Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level. Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_4031_hel_wiki.csv) Import Type: Fully Autorefresh Source Data Availability: 1998 to 2023 @@ -91,7 +91,7 @@ Fully Autorefresh:" 45 15 29 * * " (Runs at 3:45 PM on the 29th of every month). 1. import_name": "Zurich_Population_Number_Of_Birth" 2. Import Overview -Zurich population data at Province and City Level. +Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level. Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_4031_wiki.csv) Import Type: Fully Autorefresh Source Data Availability: 1998 to 2023 @@ -113,7 +113,7 @@ Fully Autorefresh:" 0 20 29 * * " (Runs at 8:00 PM on the 29th of every month). 1. import_name": "Zurich_Population_By_Origin" 2. Import Overview -Zurich population data at Province and City Level. +Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level. Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3903_hel_wiki.csv) Import Type: Fully Autorefresh Source Data Availability: 1993 to 2023 @@ -136,7 +136,7 @@ Fully Autorefresh:" 15 1 29 * * " (Runs at 1:15 AM on the 29th of every month). 1. import_name": "Zurich_Population_Number_Of_Birth_By_Sex" 2. Import Overview -Zurich population data at Province and City Level. +Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level. Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_4031_sex_wiki.csv) Import Type: Fully Autorefresh Source Data Availability: 1993 to 2023 @@ -158,7 +158,7 @@ Fully Autorefresh:" 07 19 29 * * " (Runs at 7:07 PM on the 29th of every month). 1. import_name": "Zurich_Population_By_Sex" 2. Import Overview -Zurich population data at Province and City Level. +Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level. Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3903_sex_wiki.csv) Import Type: Fully Autorefresh Source Data Availability: 1993 to 2023 diff --git a/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv b/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv index 42048306f6..3697260025 100644 --- a/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv +++ b/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv @@ -1,51 +1,51 @@ -observationAbout,observationDate,value,variableMeasured,unit -wikidataId/Q692511,2025,3252,dcid:Count_Person, -wikidataId/Q692511,2024,3260,dcid:Count_Person, -wikidataId/Q692511,2023,3343,dcid:Count_Person, -wikidataId/Q692511,2022,3330,dcid:Count_Person, -wikidataId/Q692511,2021,3275,dcid:Count_Person, -wikidataId/Q692511,2020,3317,dcid:Count_Person, -wikidataId/Q692511,2019,3307,dcid:Count_Person, -wikidataId/Q692511,2018,3267,dcid:Count_Person, -wikidataId/Q692511,2017,3266,dcid:Count_Person, -wikidataId/Q692511,2016,3226,dcid:Count_Person, -wikidataId/Q692511,2015,3197,dcid:Count_Person, -wikidataId/Q692511,2014,3236,dcid:Count_Person, -wikidataId/Q692511,2013,3194,dcid:Count_Person, -wikidataId/Q692511,2012,3164,dcid:Count_Person, -wikidataId/Q692511,2011,3140,dcid:Count_Person, -wikidataId/Q692511,2010,3099,dcid:Count_Person, -wikidataId/Q692511,2009,3110,dcid:Count_Person, -wikidataId/Q692511,2008,3051,dcid:Count_Person, -wikidataId/Q692511,2007,3079,dcid:Count_Person, -wikidataId/Q692511,2006,3097,dcid:Count_Person, -wikidataId/Q692511,2005,3081,dcid:Count_Person, -wikidataId/Q692511,2004,3115,dcid:Count_Person, -wikidataId/Q692511,2003,3149,dcid:Count_Person, -wikidataId/Q692511,2002,3128,dcid:Count_Person, -wikidataId/Q692511,2001,3103,dcid:Count_Person, -wikidataId/Q692511,2000,3109,dcid:Count_Person, -wikidataId/Q692511,1999,3104,dcid:Count_Person, -wikidataId/Q692511,1998,3050,dcid:Count_Person, -wikidataId/Q692511,1997,3083,dcid:Count_Person, -wikidataId/Q692511,1996,3078,dcid:Count_Person, -wikidataId/Q692511,1995,3049,dcid:Count_Person, -wikidataId/Q692511,1994,3089,dcid:Count_Person, -wikidataId/Q692511,1993,3133,dcid:Count_Person, -wikidataId/Q692511,1992,3186,dcid:Count_Person, -wikidataId/Q692511,1991,3173,dcid:Count_Person, -wikidataId/Q692511,1990,3145,dcid:Count_Person, -wikidataId/Q692511,1989,3161,dcid:Count_Person, -wikidataId/Q692511,1988,3187,dcid:Count_Person, -wikidataId/Q692511,1987,3218,dcid:Count_Person, -wikidataId/Q692511,1986,3177,dcid:Count_Person, -wikidataId/Q692511,1985,3181,dcid:Count_Person, -wikidataId/Q692511,1984,3277,dcid:Count_Person, -wikidataId/Q692511,1983,3304,dcid:Count_Person, -wikidataId/Q692511,1982,3356,dcid:Count_Person, -wikidataId/Q692511,1981,3501,dcid:Count_Person, -wikidataId/Q692511,1980,3539,dcid:Count_Person, -wikidataId/Q692511,1979,3551,dcid:Count_Person, -wikidataId/Q692511,1978,3710,dcid:Count_Person, -wikidataId/Q692511,1977,3802,dcid:Count_Person, -wikidataId/Q692511,1976,3836,dcid:Count_Person, +observationAbout,observationDate,value,variableMeasured +wikidataId/Q692511,2025,3252,dcid:Count_Person +wikidataId/Q692511,2024,3260,dcid:Count_Person +wikidataId/Q692511,2023,3343,dcid:Count_Person +wikidataId/Q692511,2022,3330,dcid:Count_Person +wikidataId/Q692511,2021,3275,dcid:Count_Person +wikidataId/Q692511,2020,3317,dcid:Count_Person +wikidataId/Q692511,2019,3307,dcid:Count_Person +wikidataId/Q692511,2018,3267,dcid:Count_Person +wikidataId/Q692511,2017,3266,dcid:Count_Person +wikidataId/Q692511,2016,3226,dcid:Count_Person +wikidataId/Q692511,2015,3197,dcid:Count_Person +wikidataId/Q692511,2014,3236,dcid:Count_Person +wikidataId/Q692511,2013,3194,dcid:Count_Person +wikidataId/Q692511,2012,3164,dcid:Count_Person +wikidataId/Q692511,2011,3140,dcid:Count_Person +wikidataId/Q692511,2010,3099,dcid:Count_Person +wikidataId/Q692511,2009,3110,dcid:Count_Person +wikidataId/Q692511,2008,3051,dcid:Count_Person +wikidataId/Q692511,2007,3079,dcid:Count_Person +wikidataId/Q692511,2006,3097,dcid:Count_Person +wikidataId/Q692511,2005,3081,dcid:Count_Person +wikidataId/Q692511,2004,3115,dcid:Count_Person +wikidataId/Q692511,2003,3149,dcid:Count_Person +wikidataId/Q692511,2002,3128,dcid:Count_Person +wikidataId/Q692511,2001,3103,dcid:Count_Person +wikidataId/Q692511,2000,3109,dcid:Count_Person +wikidataId/Q692511,1999,3104,dcid:Count_Person +wikidataId/Q692511,1998,3050,dcid:Count_Person +wikidataId/Q692511,1997,3083,dcid:Count_Person +wikidataId/Q692511,1996,3078,dcid:Count_Person +wikidataId/Q692511,1995,3049,dcid:Count_Person +wikidataId/Q692511,1994,3089,dcid:Count_Person +wikidataId/Q692511,1993,3133,dcid:Count_Person +wikidataId/Q692511,1992,3186,dcid:Count_Person +wikidataId/Q692511,1991,3173,dcid:Count_Person +wikidataId/Q692511,1990,3145,dcid:Count_Person +wikidataId/Q692511,1989,3161,dcid:Count_Person +wikidataId/Q692511,1988,3187,dcid:Count_Person +wikidataId/Q692511,1987,3218,dcid:Count_Person +wikidataId/Q692511,1986,3177,dcid:Count_Person +wikidataId/Q692511,1985,3181,dcid:Count_Person +wikidataId/Q692511,1984,3277,dcid:Count_Person +wikidataId/Q692511,1983,3304,dcid:Count_Person +wikidataId/Q692511,1982,3356,dcid:Count_Person +wikidataId/Q692511,1981,3501,dcid:Count_Person +wikidataId/Q692511,1980,3539,dcid:Count_Person +wikidataId/Q692511,1979,3551,dcid:Count_Person +wikidataId/Q692511,1978,3710,dcid:Count_Person +wikidataId/Q692511,1977,3802,dcid:Count_Person +wikidataId/Q692511,1976,3836,dcid:Count_Person diff --git a/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf b/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf index 73edb97d01..9b82f0f8e5 100644 --- a/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf +++ b/statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf @@ -3,5 +3,4 @@ observationAbout: C:zurich_population_bev_3240_wiki_output->observationAbout observationDate: C:zurich_population_bev_3240_wiki_output->observationDate value: C:zurich_population_bev_3240_wiki_output->value variableMeasured: C:zurich_population_bev_3240_wiki_output->variableMeasured -unit: C:zurich_population_bev_3240_wiki_output->unit typeOf: dcs:StatVarObservation diff --git a/statvar_imports/zurich/bev_3240_wiki/validation_config.json b/statvar_imports/zurich/bev_3240_wiki/validation_config.json index 66014b3369..2d7ec5a56f 100644 --- a/statvar_imports/zurich/bev_3240_wiki/validation_config.json +++ b/statvar_imports/zurich/bev_3240_wiki/validation_config.json @@ -8,6 +8,21 @@ "params": { "threshold": 0.1 } + }, + { + "rule_id": "check_max_date_consistent", + "description": "Checks if the MaxDate is the same for all StatVars.", + "validator": "MAX_DATE_CONSISTENT", + "params": {} + }, + { + "rule_id": "check_max_date_freshness", + "description": "Verifies that MaxDate meets the expected freshness threshold for the dataset.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT MAX(MaxDate) >= '2023' AS is_fresh, COUNT(*) AS row_cnt FROM stats", + "condition": "is_fresh = TRUE AND row_cnt > 0" + } } ] } diff --git a/statvar_imports/zurich/manifest.json b/statvar_imports/zurich/manifest.json index a788a25cb9..9c14b154e9 100644 --- a/statvar_imports/zurich/manifest.json +++ b/statvar_imports/zurich/manifest.json @@ -13,6 +13,7 @@ "../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=wir_2552_wiki/input_files/WIR255OD2552_rollups.csv --pv_map=wir_2552_wiki/wir_2552_wiki_pvmap.csv --config_file=wir_2552_wiki/wir_2552_wiki_metadata.csv --output_columns=observationAbout,observationDate,value,variableMeasured --output_path=wir_2552_wiki/output/zurich_population_wir_2552_wiki --output_counters=wir_2552_wiki/counters/zurich_population_wir_2552_wiki_counters.csv" ], "source_files": [ + "wir_2552_wiki/input_files/WIR255OD2552.csv", "wir_2552_wiki/input_files/WIR255OD2552_rollups.csv", "wir_2552_wiki/counters/*.csv" ], @@ -20,7 +21,7 @@ { "template_mcf": "wir_2552_wiki/output/zurich_population_wir_2552_wiki.tmcf", "cleaned_csv": "wir_2552_wiki/output/zurich_population_wir_2552_wiki.csv", - "node_mcf": "wir_2552_wiki/output/zurich_population_wir_2552_wiki_stat_vars.mcf" + "node_mcf": "wir_2552_wiki/output/*.mcf" } ], "resource_limits": { @@ -71,7 +72,8 @@ "import_inputs": [ { "template_mcf": "bev_3240_wiki/output/zurich_population_bev_3240_wiki.tmcf", - "cleaned_csv": "bev_3240_wiki/output/zurich_population_bev_3240_wiki.csv" + "cleaned_csv": "bev_3240_wiki/output/zurich_population_bev_3240_wiki.csv", + "node_mcf": "bev_3240_wiki/output/*.mcf" } ], "resource_limits": { diff --git a/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py b/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py index 452fb21b13..84f4625ca9 100644 --- a/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py +++ b/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py @@ -1,50 +1,132 @@ #!/usr/bin/env python3 -"""Preprocesses Zurich WIR255OD2552 dataset to filter total roll-up rows (Alle Rechtsformen and Alle Betriebsgrössen).""" +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Preprocesses Zurich WIR255OD2552 dataset to filter total roll-up rows. + +Filters total rows (Alle Rechtsformen and Alle Betriebsgrössen) where +RechtsformSort == 0 and BetriebsgrSort == 0, and coerces numeric metric +columns to numbers (converting markers such as 'K' to NaN). +""" import os +import sys +from absl import app +from absl import flags +from absl import logging import pandas as pd CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -INPUT_RAW = os.path.join(CURRENT_DIR, 'input_files', 'WIR255OD2552.csv') -OUTPUT_ROLLUPS = os.path.join(CURRENT_DIR, 'input_files', 'WIR255OD2552_rollups.csv') +DEFAULT_INPUT_RAW = os.path.join(CURRENT_DIR, 'input_files', 'WIR255OD2552.csv') +DEFAULT_OUTPUT_ROLLUPS = os.path.join(CURRENT_DIR, 'input_files', + 'WIR255OD2552_rollups.csv') -def main(): - if not os.path.exists(INPUT_RAW): - raise FileNotFoundError(f"{INPUT_RAW} not found.") +FLAGS = flags.FLAGS +flags.DEFINE_string('input_csv', DEFAULT_INPUT_RAW, + 'Path to raw input CSV file.') +flags.DEFINE_string('output_csv', DEFAULT_OUTPUT_ROLLUPS, + 'Path to output rollups CSV file.') - try: - df = pd.read_csv(INPUT_RAW, encoding='utf-8') - except UnicodeDecodeError: - df = pd.read_csv(INPUT_RAW, encoding='iso-8859-1') +REQUIRED_FILTER_COLS = ['RechtsformSort', 'BetriebsgrSort'] +NUMERIC_VAL_COLS = [ + 'Arbeitsstaetten', 'AnzBesch', 'AnzBeschW', 'AnzBeschM', 'AnzVZA', + 'AnzVZAW', 'AnzVZAM' +] + + +def process_rollups(df: pd.DataFrame) -> pd.DataFrame: + """Filters total roll-up rows and coerces metric columns to numeric. + + Args: + df: Input pandas DataFrame containing raw Zurich workplace data. + Returns: + Filtered pandas DataFrame containing only total rollup rows. + + Raises: + ValueError: If input DataFrame is empty or no rows match filter. + KeyError: If required filter columns are missing. + """ if df.empty: - raise ValueError(f"Input file {INPUT_RAW} is empty.") + raise ValueError("Input DataFrame is empty.") - required_filter_cols = ['RechtsformSort', 'BetriebsgrSort'] - missing_cols = [col for col in required_filter_cols if col not in df.columns] + missing_cols = [ + col for col in REQUIRED_FILTER_COLS if col not in df.columns + ] if missing_cols: raise KeyError( - f"Input file {INPUT_RAW} is missing required filter columns: {missing_cols}. " - f"Available columns: {list(df.columns)}" - ) + f"Input DataFrame is missing required filter columns: {missing_cols}. " + f"Available columns: {list(df.columns)}") - # Keep only the total rows (Alle Rechtsformen and Alle Betriebsgrössen) - df_rollups = df[(df['RechtsformSort'] == 0) & (df['BetriebsgrSort'] == 0)].copy() + # Keep only the total rows (Alle Rechtsformen and Alle Betriebsgrössen). + # Coerce sort columns to numeric so string or numeric '0' matches reliably. + rechtsform_sort = pd.to_numeric(df['RechtsformSort'], errors='coerce') + betriebsgr_sort = pd.to_numeric(df['BetriebsgrSort'], errors='coerce') + df_rollups = df[(rechtsform_sort == 0) & (betriebsgr_sort == 0)].copy() if df_rollups.empty: raise ValueError( - f"No rows matched filter (RechtsformSort == 0 & BetriebsgrSort == 0) in {INPUT_RAW}." + "No rows matched filter (RechtsformSort == 0 & BetriebsgrSort == 0)." ) # Convert numeric columns, converting non-numeric markers (e.g. 'K') to NaN/numeric - val_cols = ['Arbeitsstaetten', 'AnzBesch', 'AnzBeschW', 'AnzBeschM', 'AnzVZA', 'AnzVZAW', 'AnzVZAM'] - for col in val_cols: + for col in NUMERIC_VAL_COLS: if col in df_rollups.columns: df_rollups[col] = pd.to_numeric(df_rollups[col], errors='coerce') - os.makedirs(os.path.dirname(OUTPUT_ROLLUPS), exist_ok=True) - df_rollups.to_csv(OUTPUT_ROLLUPS, index=False, encoding='utf-8') - print(f"Successfully generated {OUTPUT_ROLLUPS} with {len(df_rollups)} rows.") + return df_rollups + + +def generate_rollups(input_csv: str, output_csv: str) -> pd.DataFrame: + """Reads raw CSV, processes rollups, and writes the output CSV file. + + Args: + input_csv: Path to input CSV file. + output_csv: Path to output CSV file. + + Returns: + Processed pandas DataFrame written to output_csv. + + Raises: + FileNotFoundError: If input_csv does not exist. + ValueError: If input_csv is empty. + """ + if not os.path.exists(input_csv): + raise FileNotFoundError(f"Input file not found: {input_csv}") + if os.path.getsize(input_csv) == 0: + raise ValueError(f"Input file is empty: {input_csv}") + + try: + df = pd.read_csv(input_csv, encoding='utf-8') + except UnicodeDecodeError: + df = pd.read_csv(input_csv, encoding='iso-8859-1') + + df_rollups = process_rollups(df) + + output_dir = os.path.dirname(output_csv) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + + df_rollups.to_csv(output_csv, index=False, encoding='utf-8') + logging.info("Successfully generated %s with %d rows.", output_csv, + len(df_rollups)) + return df_rollups + + +def main(argv): + del argv # Unused. + generate_rollups(FLAGS.input_csv, FLAGS.output_csv) + if __name__ == '__main__': - main() + app.run(main) diff --git a/statvar_imports/zurich/wir_2552_wiki/generate_rollups_test.py b/statvar_imports/zurich/wir_2552_wiki/generate_rollups_test.py new file mode 100644 index 0000000000..a2c5c321b9 --- /dev/null +++ b/statvar_imports/zurich/wir_2552_wiki/generate_rollups_test.py @@ -0,0 +1,157 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Hermetic unit tests for Zurich wir_2552_wiki generate_rollups module.""" + +import os +import sys +import tempfile +import unittest +import numpy as np +import pandas as pd + +# Add module directory to sys.path +_MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) +if _MODULE_DIR not in sys.path: + sys.path.insert(0, _MODULE_DIR) + +from generate_rollups import generate_rollups, process_rollups + + +class GenerateRollupsTest(unittest.TestCase): + """Unit test suite for generate_rollups preprocessor.""" + + def setUp(self): + super().setUp() + self.sample_raw_data = pd.DataFrame({ + 'Jahr': [2020, 2020, 2020, 2020], + 'RaumSort': [1, 1, 1, 1], + 'RaumLang': [ + 'Ganze Stadt', 'Ganze Stadt', 'Ganze Stadt', 'Ganze Stadt' + ], + 'RechtsformSort': [0, 1, 0, 2], + 'RechtsformLang': [ + 'Alle Rechtsformen', 'Öffentliches Recht', 'Alle Rechtsformen', + 'Privates Recht' + ], + 'BetriebsgrSort': [0, 0, 1, 2], + 'BetriebsgrLang': [ + 'Alle Betriebsgrössen', 'Alle Betriebsgrössen', 'Mikrobetriebe', + 'Grossbetriebe' + ], + 'Arbeitsstaetten': ['39747', '500', '1200', '100'], + 'AnzBesch': ['443621', 'K', '5000', '25000'], + 'AnzBeschW': ['202970', '250', '2500', '12000'], + 'AnzBeschM': ['240651', '250', '2500', '13000'], + 'AnzVZA': ['339750', '400', '4000', '20000'], + 'AnzVZAW': ['136406', '200', '2000', '10000'], + 'AnzVZAM': ['203349', '200', '2000', '10000'] + }) + + def test_process_rollups_filters_total_rows(self): + """Verifies that only total rollup rows (RechtsformSort == 0 & BetriebsgrSort == 0) are retained.""" + result_df = process_rollups(self.sample_raw_data) + self.assertEqual(len(result_df), 1) + self.assertEqual(result_df.iloc[0]['RechtsformSort'], 0) + self.assertEqual(result_df.iloc[0]['BetriebsgrSort'], 0) + self.assertEqual(result_df.iloc[0]['RechtsformLang'], + 'Alle Rechtsformen') + self.assertEqual(result_df.iloc[0]['BetriebsgrLang'], + 'Alle Betriebsgrössen') + + def test_process_rollups_numeric_coercion(self): + """Verifies that metric columns are converted to numeric and markers like 'K' become NaN.""" + data = pd.DataFrame({ + 'RechtsformSort': [0, 0], + 'BetriebsgrSort': [0, 0], + 'Arbeitsstaetten': ['100', '200'], + 'AnzBesch': ['1500', 'K'], + 'AnzBeschW': [500, 'K'], + 'AnzBeschM': ['1000', 100], + 'AnzVZA': ['1200.5', 'K'], + 'AnzVZAW': ['600.25', '300'], + 'AnzVZAM': ['600.25', 'K'] + }) + result_df = process_rollups(data) + self.assertEqual(len(result_df), 2) + self.assertEqual(result_df.iloc[0]['AnzBesch'], 1500.0) + self.assertTrue(pd.isna(result_df.iloc[1]['AnzBesch'])) + self.assertTrue(pd.isna(result_df.iloc[1]['AnzBeschW'])) + self.assertEqual(result_df.iloc[0]['AnzVZA'], 1200.5) + self.assertTrue(pd.isna(result_df.iloc[1]['AnzVZA'])) + self.assertEqual(result_df.iloc[1]['AnzVZAW'], 300.0) + + def test_process_rollups_empty_dataframe_raises(self): + """Verifies that an empty DataFrame raises ValueError.""" + with self.assertRaises(ValueError): + process_rollups(pd.DataFrame()) + + def test_process_rollups_missing_columns_raises(self): + """Verifies that missing filter columns raise KeyError.""" + df_missing = pd.DataFrame({'RechtsformSort': [0]}) + with self.assertRaises(KeyError): + process_rollups(df_missing) + + def test_process_rollups_no_matching_rows_raises(self): + """Verifies that data with no total rows (0, 0) raises ValueError.""" + df_no_match = pd.DataFrame({ + 'RechtsformSort': [1, 2], + 'BetriebsgrSort': [1, 2] + }) + with self.assertRaises(ValueError): + process_rollups(df_no_match) + + def test_generate_rollups_file_io_success(self): + """Verifies end-to-end file reading, processing, and output generation.""" + with tempfile.TemporaryDirectory() as tmp_dir: + input_csv = os.path.join(tmp_dir, 'input.csv') + output_csv = os.path.join(tmp_dir, 'output_dir', + 'output_rollups.csv') + + self.sample_raw_data.to_csv(input_csv, + index=False, + encoding='utf-8') + result_df = generate_rollups(input_csv, output_csv) + + self.assertTrue(os.path.exists(output_csv)) + self.assertEqual(len(result_df), 1) + + read_back = pd.read_csv(output_csv) + self.assertEqual(len(read_back), 1) + self.assertEqual(read_back.iloc[0]['Arbeitsstaetten'], 39747) + + def test_generate_rollups_file_not_found(self): + """Verifies that non-existent input file raises FileNotFoundError.""" + with self.assertRaises(FileNotFoundError): + generate_rollups('/non/existent/path/file.csv', '/tmp/out.csv') + + def test_generate_rollups_empty_file_raises(self): + """Verifies that an empty input file raises ValueError.""" + with tempfile.NamedTemporaryFile(suffix='.csv') as tmp_file: + with self.assertRaises(ValueError): + generate_rollups(tmp_file.name, '/tmp/out.csv') + + def test_process_rollups_string_sort_columns(self): + """Verifies that sort columns formatted as strings are matched correctly.""" + data = pd.DataFrame({ + 'RechtsformSort': ['0', '1'], + 'BetriebsgrSort': ['0', '0'], + 'Arbeitsstaetten': ['100', '200'] + }) + result_df = process_rollups(data) + self.assertEqual(len(result_df), 1) + self.assertEqual(result_df.iloc[0]['Arbeitsstaetten'], 100.0) + + +if __name__ == '__main__': + unittest.main() diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_raw_input.csv b/statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_raw_input.csv new file mode 100644 index 0000000000..7521ff0e09 --- /dev/null +++ b/statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_raw_input.csv @@ -0,0 +1,57 @@ +Jahr,RaumSort,RaumLang,RechtsformSort,RechtsformLang,BetriebsgrSort,BetriebsgrLang,Arbeitsstaetten,AnzBesch,AnzBeschW,AnzBeschM,AnzVZA,AnzVZAW,AnzVZAM +2011,1,Ganze Stadt,0,Alle Rechtsformen,0,Alle Betriebsgrössen,39747,443621,202970,240651,339750,136406,203349 +2011,1,Ganze Stadt,1,Öffentliches Recht,0,Alle Betriebsgrössen,120,45000,20000,25000,38000,16000,22000 +2011,1,Ganze Stadt,2,Privatrechtliche Unternehmen,0,Alle Betriebsgrössen,35000,350000,160000,190000,270000,110000,160000 +2011,1,Ganze Stadt,0,Alle Rechtsformen,1,Mikrobetriebe (unter 10),30000,80000,35000,45000,60000,25000,35000 +2011,1,Ganze Stadt,0,Alle Rechtsformen,2,Kleinbetriebe (10-49),7000,120000,55000,65000,95000,40000,55000 +2011,10,Kreis 1,0,Alle Rechtsformen,0,Alle Betriebsgrössen,5249,68627,34662,33965,53138,24338,28799 +2011,10,Kreis 1,1,Öffentliches Recht,0,Alle Betriebsgrössen,50,12000,5000,7000,10000,4000,6000 +2011,10,Kreis 1,0,Alle Rechtsformen,1,Mikrobetriebe (unter 10),3500,10000,4500,5500,8000,3500,4500 +2011,11,Rathaus,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1522,10180,5492,4688,7205,3493,3710 +2011,12,Hochschulen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,700,13203,7025,6178,9090,4376,4714 +2011,13,Lindenhof,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1193,15160,7649,7511,12213,5602,6608 +2011,14,City,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1834,30084,14496,15588,24633,10861,13768 +2011,20,Kreis 2,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3690,42295,19218,23077,32625,12786,19844 +2011,21,Wollishofen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1050,7578,3396,4182,5865,2271,3594 +2011,23,Leimbach,0,Alle Rechtsformen,0,Alle Betriebsgrössen,177,640,340,300,460,217,246 +2011,24,Enge,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2463,34077,15482,18595,26297,10298,16002 +2011,30,Kreis 3,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3716,41539,18494,23045,32146,12425,19724 +2011,31,Alt-Wiedikon,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1668,25737,10281,15456,20428,6962,13465 +2011,33,Friesenberg,0,Alle Rechtsformen,0,Alle Betriebsgrössen,358,4510,3113,1397,3523,2311,1209 +2011,34,Sihlfeld,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1690,11292,5100,6192,8196,3148,5049 +2011,40,Kreis 4,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3892,36061,15217,20844,27489,10031,17459 +2011,41,Werd,0,Alle Rechtsformen,0,Alle Betriebsgrössen,984,9023,4202,4821,6753,2834,3919 +2011,42,Langstrasse,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2066,19957,8639,11318,15238,5743,9495 +2011,44,Hard,0,Alle Rechtsformen,0,Alle Betriebsgrössen,842,7081,2376,4705,5497,1455,4045 +2011,50,Kreis 5,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2552,44156,16755,27401,35887,11991,23902 +2011,51,Gewerbeschule,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1363,15655,7201,8454,11596,4844,6753 +2011,52,Escher Wyss,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1189,28501,9554,18947,24293,7144,17146 +2011,60,Kreis 6,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2961,25736,12597,13139,18409,8037,10367 +2011,61,Unterstrass,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1931,15235,7380,7855,11198,4786,6413 +2011,63,Oberstrass,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1030,10501,5217,5284,7210,3253,3956 +2011,70,Kreis 7,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3758,31803,19099,12704,23623,13268,10356 +2011,71,Fluntern,0,Alle Rechtsformen,0,Alle Betriebsgrössen,838,14560,8961,5599,11196,6607,4590 +2011,72,Hottingen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1701,11377,6669,4708,8268,4448,3822 +2011,73,Hirslanden,0,Alle Rechtsformen,0,Alle Betriebsgrössen,715,3824,2212,1612,2678,1383,1296 +2011,74,Witikon,0,Alle Rechtsformen,0,Alle Betriebsgrössen,504,2042,1257,785,1483,833,647 +2011,80,Kreis 8,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3675,28912,15351,13561,21358,10330,11025 +2011,81,Seefeld,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1526,9963,4975,4988,7537,3393,4146 +2011,82,Mühlebach,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1503,9161,4556,4605,6891,3024,3869 +2011,83,Weinegg,0,Alle Rechtsformen,0,Alle Betriebsgrössen,646,9788,5820,3968,6925,3913,3011 +2011,90,Kreis 9,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3277,48818,18683,30135,38569,12314,26256 +2011,91,Albisrieden,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1024,7797,3310,4487,6040,2102,3940 +2011,92,Altstetten,0,Alle Rechtsformen,0,Alle Betriebsgrössen,2253,41021,15373,25648,32530,10215,22315 +2011,100,Kreis 10,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1983,15272,7707,7565,10589,4753,5836 +2011,101,Höngg,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1031,8233,3643,4590,5796,2281,3515 +2011,102,Wipkingen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,952,7039,4064,2975,4795,2475,2322 +2011,110,Kreis 11,0,Alle Rechtsformen,0,Alle Betriebsgrössen,3902,52782,21371,31411,40609,13940,26668 +2011,111,Affoltern,0,Alle Rechtsformen,0,Alle Betriebsgrössen,800,3553,1675,1878,2671,1084,1586 +2011,115,Oerlikon,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1767,23926,10162,13764,18381,6569,11812 +2011,119,Seebach,0,Alle Rechtsformen,0,Alle Betriebsgrössen,1335,25303,9534,15769,19553,6288,13267 +2011,120,Kreis 12,0,Alle Rechtsformen,0,Alle Betriebsgrössen,945,6868,3415,3453,4762,1932,2828 +2011,121,Saatlen,0,Alle Rechtsformen,0,Alle Betriebsgrössen,183,1516,493,1023,1197,319,879 +2011,122,Schwamendingen-Mitte,0,Alle Rechtsformen,0,Alle Betriebsgrössen,464,3690,2029,1661,2305,982,1325 +2011,123,Hirzenbach,0,Alle Rechtsformen,0,Alle Betriebsgrössen,298,1662,893,769,1258,636,624 +2011,990,Kreis Unbekannt,0,Alle Rechtsformen,0,Alle Betriebsgrössen,147,752,401,351,549,258,292 +2011,999,Quartier Unbekannt,0,Alle Rechtsformen,0,Alle Betriebsgrössen,147,752,401,351,549,258,292 +2012,1,Ganze Stadt,0,Alle Rechtsformen,0,Alle Betriebsgrössen,40362,448474,205879,242595,343562,138749,204816 diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.csv b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.csv index 3475920ecb..97ecdb447e 100644 --- a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.csv +++ b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.csv @@ -1,337 +1,337 @@ -observationAbout,observationDate,value,variableMeasured,unit -wikidataId/Q72,2011,39747,dcid:Count_Company, -wikidataId/Q72,2011,443621,dcid:Count_Person_Employed, -wikidataId/Q72,2011,202970,dcid:Count_Person_Employed_Female, -wikidataId/Q72,2011,240651,dcid:Count_Person_Employed_Male, -wikidataId/Q72,2011,339750,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q72,2011,136406,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q72,2011,203349,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q445559,2011,5249,dcid:Count_Company, -wikidataId/Q445559,2011,68627,dcid:Count_Person_Employed, -wikidataId/Q445559,2011,34662,dcid:Count_Person_Employed_Female, -wikidataId/Q445559,2011,33965,dcid:Count_Person_Employed_Male, -wikidataId/Q445559,2011,53138,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q445559,2011,24338,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q445559,2011,28799,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q692511,2011,1522,dcid:Count_Company, -wikidataId/Q692511,2011,10180,dcid:Count_Person_Employed, -wikidataId/Q692511,2011,5492,dcid:Count_Person_Employed_Female, -wikidataId/Q692511,2011,4688,dcid:Count_Person_Employed_Male, -wikidataId/Q692511,2011,7205,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q692511,2011,3493,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q692511,2011,3710,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q39240,2011,700,dcid:Count_Company, -wikidataId/Q39240,2011,13203,dcid:Count_Person_Employed, -wikidataId/Q39240,2011,7025,dcid:Count_Person_Employed_Female, -wikidataId/Q39240,2011,6178,dcid:Count_Person_Employed_Male, -wikidataId/Q39240,2011,9090,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q39240,2011,4376,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q39240,2011,4714,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q10987378,2011,1193,dcid:Count_Company, -wikidataId/Q10987378,2011,15160,dcid:Count_Person_Employed, -wikidataId/Q10987378,2011,7649,dcid:Count_Person_Employed_Female, -wikidataId/Q10987378,2011,7511,dcid:Count_Person_Employed_Male, -wikidataId/Q10987378,2011,12213,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q10987378,2011,5602,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q10987378,2011,6608,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q1093831,2011,1834,dcid:Count_Company, -wikidataId/Q1093831,2011,30084,dcid:Count_Person_Employed, -wikidataId/Q1093831,2011,14496,dcid:Count_Person_Employed_Female, -wikidataId/Q1093831,2011,15588,dcid:Count_Person_Employed_Male, -wikidataId/Q1093831,2011,24633,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q1093831,2011,10861,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q1093831,2011,13768,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q456153,2011,3690,dcid:Count_Company, -wikidataId/Q456153,2011,42295,dcid:Count_Person_Employed, -wikidataId/Q456153,2011,19218,dcid:Count_Person_Employed_Female, -wikidataId/Q456153,2011,23077,dcid:Count_Person_Employed_Male, -wikidataId/Q456153,2011,32625,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q456153,2011,12786,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q456153,2011,19844,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q642353,2011,1050,dcid:Count_Company, -wikidataId/Q642353,2011,7578,dcid:Count_Person_Employed, -wikidataId/Q642353,2011,3396,dcid:Count_Person_Employed_Female, -wikidataId/Q642353,2011,4182,dcid:Count_Person_Employed_Male, -wikidataId/Q642353,2011,5865,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q642353,2011,2271,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q642353,2011,3594,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q691367,2011,177,dcid:Count_Company, -wikidataId/Q691367,2011,640,dcid:Count_Person_Employed, -wikidataId/Q691367,2011,340,dcid:Count_Person_Employed_Female, -wikidataId/Q691367,2011,300,dcid:Count_Person_Employed_Male, -wikidataId/Q691367,2011,460,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q691367,2011,217,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q691367,2011,246,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q648218,2011,2463,dcid:Count_Company, -wikidataId/Q648218,2011,34077,dcid:Count_Person_Employed, -wikidataId/Q648218,2011,15482,dcid:Count_Person_Employed_Female, -wikidataId/Q648218,2011,18595,dcid:Count_Person_Employed_Male, -wikidataId/Q648218,2011,26297,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q648218,2011,10298,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q648218,2011,16002,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q675017,2011,3716,dcid:Count_Company, -wikidataId/Q675017,2011,41539,dcid:Count_Person_Employed, -wikidataId/Q675017,2011,18494,dcid:Count_Person_Employed_Female, -wikidataId/Q675017,2011,23045,dcid:Count_Person_Employed_Male, -wikidataId/Q675017,2011,32146,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q675017,2011,12425,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q675017,2011,19724,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q433012,2011,1668,dcid:Count_Company, -wikidataId/Q433012,2011,25737,dcid:Count_Person_Employed, -wikidataId/Q433012,2011,10281,dcid:Count_Person_Employed_Female, -wikidataId/Q433012,2011,15456,dcid:Count_Person_Employed_Male, -wikidataId/Q433012,2011,20428,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q433012,2011,6962,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q433012,2011,13465,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q693357,2011,358,dcid:Count_Company, -wikidataId/Q693357,2011,4510,dcid:Count_Person_Employed, -wikidataId/Q693357,2011,3113,dcid:Count_Person_Employed_Female, -wikidataId/Q693357,2011,1397,dcid:Count_Person_Employed_Male, -wikidataId/Q693357,2011,3523,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q693357,2011,2311,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q693357,2011,1209,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q370104,2011,1690,dcid:Count_Company, -wikidataId/Q370104,2011,11292,dcid:Count_Person_Employed, -wikidataId/Q370104,2011,5100,dcid:Count_Person_Employed_Female, -wikidataId/Q370104,2011,6192,dcid:Count_Person_Employed_Male, -wikidataId/Q370104,2011,8196,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q370104,2011,3148,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q370104,2011,5049,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q677133,2011,3892,dcid:Count_Company, -wikidataId/Q677133,2011,36061,dcid:Count_Person_Employed, -wikidataId/Q677133,2011,15217,dcid:Count_Person_Employed_Female, -wikidataId/Q677133,2011,20844,dcid:Count_Person_Employed_Male, -wikidataId/Q677133,2011,27489,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q677133,2011,10031,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q677133,2011,17459,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q531899,2011,984,dcid:Count_Company, -wikidataId/Q531899,2011,9023,dcid:Count_Person_Employed, -wikidataId/Q531899,2011,4202,dcid:Count_Person_Employed_Female, -wikidataId/Q531899,2011,4821,dcid:Count_Person_Employed_Male, -wikidataId/Q531899,2011,6753,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q531899,2011,2834,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q531899,2011,3919,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q1805410,2011,2066,dcid:Count_Company, -wikidataId/Q1805410,2011,19957,dcid:Count_Person_Employed, -wikidataId/Q1805410,2011,8639,dcid:Count_Person_Employed_Female, -wikidataId/Q1805410,2011,11318,dcid:Count_Person_Employed_Male, -wikidataId/Q1805410,2011,15238,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q1805410,2011,5743,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q1805410,2011,9495,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q870084,2011,842,dcid:Count_Company, -wikidataId/Q870084,2011,7081,dcid:Count_Person_Employed, -wikidataId/Q870084,2011,2376,dcid:Count_Person_Employed_Female, -wikidataId/Q870084,2011,4705,dcid:Count_Person_Employed_Male, -wikidataId/Q870084,2011,5497,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q870084,2011,1455,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q870084,2011,4045,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q460885,2011,2552,dcid:Count_Company, -wikidataId/Q460885,2011,44156,dcid:Count_Person_Employed, -wikidataId/Q460885,2011,16755,dcid:Count_Person_Employed_Female, -wikidataId/Q460885,2011,27401,dcid:Count_Person_Employed_Male, -wikidataId/Q460885,2011,35887,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q460885,2011,11991,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q460885,2011,23902,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q693413,2011,1363,dcid:Count_Company, -wikidataId/Q693413,2011,15655,dcid:Count_Person_Employed, -wikidataId/Q693413,2011,7201,dcid:Count_Person_Employed_Female, -wikidataId/Q693413,2011,8454,dcid:Count_Person_Employed_Male, -wikidataId/Q693413,2011,11596,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q693413,2011,4844,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q693413,2011,6753,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q687052,2011,1189,dcid:Count_Company, -wikidataId/Q687052,2011,28501,dcid:Count_Person_Employed, -wikidataId/Q687052,2011,9554,dcid:Count_Person_Employed_Female, -wikidataId/Q687052,2011,18947,dcid:Count_Person_Employed_Male, -wikidataId/Q687052,2011,24293,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q687052,2011,7144,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q687052,2011,17146,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q456170,2011,2961,dcid:Count_Company, -wikidataId/Q456170,2011,25736,dcid:Count_Person_Employed, -wikidataId/Q456170,2011,12597,dcid:Count_Person_Employed_Female, -wikidataId/Q456170,2011,13139,dcid:Count_Person_Employed_Male, -wikidataId/Q456170,2011,18409,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q456170,2011,8037,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q456170,2011,10367,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q656446,2011,1931,dcid:Count_Company, -wikidataId/Q656446,2011,15235,dcid:Count_Person_Employed, -wikidataId/Q656446,2011,7380,dcid:Count_Person_Employed_Female, -wikidataId/Q656446,2011,7855,dcid:Count_Person_Employed_Male, -wikidataId/Q656446,2011,11198,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q656446,2011,4786,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q656446,2011,6413,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q693483,2011,1030,dcid:Count_Company, -wikidataId/Q693483,2011,10501,dcid:Count_Person_Employed, -wikidataId/Q693483,2011,5217,dcid:Count_Person_Employed_Female, -wikidataId/Q693483,2011,5284,dcid:Count_Person_Employed_Male, -wikidataId/Q693483,2011,7210,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q693483,2011,3253,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q693483,2011,3956,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q456149,2011,3758,dcid:Count_Company, -wikidataId/Q456149,2011,31803,dcid:Count_Person_Employed, -wikidataId/Q456149,2011,19099,dcid:Count_Person_Employed_Female, -wikidataId/Q456149,2011,12704,dcid:Count_Person_Employed_Male, -wikidataId/Q456149,2011,23623,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q456149,2011,13268,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q456149,2011,10356,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q693269,2011,838,dcid:Count_Company, -wikidataId/Q693269,2011,14560,dcid:Count_Person_Employed, -wikidataId/Q693269,2011,8961,dcid:Count_Person_Employed_Female, -wikidataId/Q693269,2011,5599,dcid:Count_Person_Employed_Male, -wikidataId/Q693269,2011,11196,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q693269,2011,6607,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q693269,2011,4590,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q693454,2011,1701,dcid:Count_Company, -wikidataId/Q693454,2011,11377,dcid:Count_Person_Employed, -wikidataId/Q693454,2011,6669,dcid:Count_Person_Employed_Female, -wikidataId/Q693454,2011,4708,dcid:Count_Person_Employed_Male, -wikidataId/Q693454,2011,8268,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q693454,2011,4448,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q693454,2011,3822,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q476940,2011,715,dcid:Count_Company, -wikidataId/Q476940,2011,3824,dcid:Count_Person_Employed, -wikidataId/Q476940,2011,2212,dcid:Count_Person_Employed_Female, -wikidataId/Q476940,2011,1612,dcid:Count_Person_Employed_Male, -wikidataId/Q476940,2011,2678,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q476940,2011,1383,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q476940,2011,1296,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q392079,2011,504,dcid:Count_Company, -wikidataId/Q392079,2011,2042,dcid:Count_Person_Employed, -wikidataId/Q392079,2011,1257,dcid:Count_Person_Employed_Female, -wikidataId/Q392079,2011,785,dcid:Count_Person_Employed_Male, -wikidataId/Q392079,2011,1483,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q392079,2011,833,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q392079,2011,647,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q677710,2011,3675,dcid:Count_Company, -wikidataId/Q677710,2011,28912,dcid:Count_Person_Employed, -wikidataId/Q677710,2011,15351,dcid:Count_Person_Employed_Female, -wikidataId/Q677710,2011,13561,dcid:Count_Person_Employed_Male, -wikidataId/Q677710,2011,21358,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q677710,2011,10330,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q677710,2011,11025,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q692773,2011,1526,dcid:Count_Company, -wikidataId/Q692773,2011,9963,dcid:Count_Person_Employed, -wikidataId/Q692773,2011,4975,dcid:Count_Person_Employed_Female, -wikidataId/Q692773,2011,4988,dcid:Count_Person_Employed_Male, -wikidataId/Q692773,2011,7537,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q692773,2011,3393,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q692773,2011,4146,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q693397,2011,1503,dcid:Count_Company, -wikidataId/Q693397,2011,9161,dcid:Count_Person_Employed, -wikidataId/Q693397,2011,4556,dcid:Count_Person_Employed_Female, -wikidataId/Q693397,2011,4605,dcid:Count_Person_Employed_Male, -wikidataId/Q693397,2011,6891,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q693397,2011,3024,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q693397,2011,3869,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q693321,2011,646,dcid:Count_Company, -wikidataId/Q693321,2011,9788,dcid:Count_Person_Employed, -wikidataId/Q693321,2011,5820,dcid:Count_Person_Employed_Female, -wikidataId/Q693321,2011,3968,dcid:Count_Person_Employed_Male, -wikidataId/Q693321,2011,6925,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q693321,2011,3913,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q693321,2011,3011,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q456159,2011,3277,dcid:Count_Company, -wikidataId/Q456159,2011,48818,dcid:Count_Person_Employed, -wikidataId/Q456159,2011,18683,dcid:Count_Person_Employed_Female, -wikidataId/Q456159,2011,30135,dcid:Count_Person_Employed_Male, -wikidataId/Q456159,2011,38569,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q456159,2011,12314,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q456159,2011,26256,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q80797,2011,1024,dcid:Count_Company, -wikidataId/Q80797,2011,7797,dcid:Count_Person_Employed, -wikidataId/Q80797,2011,3310,dcid:Count_Person_Employed_Female, -wikidataId/Q80797,2011,4487,dcid:Count_Person_Employed_Male, -wikidataId/Q80797,2011,6040,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q80797,2011,2102,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q80797,2011,3940,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q445711,2011,2253,dcid:Count_Company, -wikidataId/Q445711,2011,41021,dcid:Count_Person_Employed, -wikidataId/Q445711,2011,15373,dcid:Count_Person_Employed_Female, -wikidataId/Q445711,2011,25648,dcid:Count_Person_Employed_Male, -wikidataId/Q445711,2011,32530,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q445711,2011,10215,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q445711,2011,22315,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q456175,2011,1983,dcid:Count_Company, -wikidataId/Q456175,2011,15272,dcid:Count_Person_Employed, -wikidataId/Q456175,2011,7707,dcid:Count_Person_Employed_Female, -wikidataId/Q456175,2011,7565,dcid:Count_Person_Employed_Male, -wikidataId/Q456175,2011,10589,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q456175,2011,4753,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q456175,2011,5836,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q455496,2011,1031,dcid:Count_Company, -wikidataId/Q455496,2011,8233,dcid:Count_Person_Employed, -wikidataId/Q455496,2011,3643,dcid:Count_Person_Employed_Female, -wikidataId/Q455496,2011,4590,dcid:Count_Person_Employed_Male, -wikidataId/Q455496,2011,5796,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q455496,2011,2281,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q455496,2011,3515,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q678030,2011,952,dcid:Count_Company, -wikidataId/Q678030,2011,7039,dcid:Count_Person_Employed, -wikidataId/Q678030,2011,4064,dcid:Count_Person_Employed_Female, -wikidataId/Q678030,2011,2975,dcid:Count_Person_Employed_Male, -wikidataId/Q678030,2011,4795,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q678030,2011,2475,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q678030,2011,2322,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q870098,2011,3902,dcid:Count_Company, -wikidataId/Q870098,2011,52782,dcid:Count_Person_Employed, -wikidataId/Q870098,2011,21371,dcid:Count_Person_Employed_Female, -wikidataId/Q870098,2011,31411,dcid:Count_Person_Employed_Male, -wikidataId/Q870098,2011,40609,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q870098,2011,13940,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q870098,2011,26668,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q382903,2011,800,dcid:Count_Company, -wikidataId/Q382903,2011,3553,dcid:Count_Person_Employed, -wikidataId/Q382903,2011,1675,dcid:Count_Person_Employed_Female, -wikidataId/Q382903,2011,1878,dcid:Count_Person_Employed_Male, -wikidataId/Q382903,2011,2671,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q382903,2011,1084,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q382903,2011,1586,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q167179,2011,1767,dcid:Count_Company, -wikidataId/Q167179,2011,23926,dcid:Count_Person_Employed, -wikidataId/Q167179,2011,10162,dcid:Count_Person_Employed_Female, -wikidataId/Q167179,2011,13764,dcid:Count_Person_Employed_Male, -wikidataId/Q167179,2011,18381,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q167179,2011,6569,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q167179,2011,11812,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q276792,2011,1335,dcid:Count_Company, -wikidataId/Q276792,2011,25303,dcid:Count_Person_Employed, -wikidataId/Q276792,2011,9534,dcid:Count_Person_Employed_Female, -wikidataId/Q276792,2011,15769,dcid:Count_Person_Employed_Male, -wikidataId/Q276792,2011,19553,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q276792,2011,6288,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q276792,2011,13267,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q657525,2011,945,dcid:Count_Company, -wikidataId/Q657525,2011,6868,dcid:Count_Person_Employed, -wikidataId/Q657525,2011,3415,dcid:Count_Person_Employed_Female, -wikidataId/Q657525,2011,3453,dcid:Count_Person_Employed_Male, -wikidataId/Q657525,2011,4762,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q657525,2011,1932,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q657525,2011,2828,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q652455,2011,183,dcid:Count_Company, -wikidataId/Q652455,2011,1516,dcid:Count_Person_Employed, -wikidataId/Q652455,2011,493,dcid:Count_Person_Employed_Female, -wikidataId/Q652455,2011,1023,dcid:Count_Person_Employed_Male, -wikidataId/Q652455,2011,1197,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q652455,2011,319,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q652455,2011,879,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q692728,2011,464,dcid:Count_Company, -wikidataId/Q692728,2011,3690,dcid:Count_Person_Employed, -wikidataId/Q692728,2011,2029,dcid:Count_Person_Employed_Female, -wikidataId/Q692728,2011,1661,dcid:Count_Person_Employed_Male, -wikidataId/Q692728,2011,2305,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q692728,2011,982,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q692728,2011,1325,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q693374,2011,298,dcid:Count_Company, -wikidataId/Q693374,2011,1662,dcid:Count_Person_Employed, -wikidataId/Q693374,2011,893,dcid:Count_Person_Employed_Female, -wikidataId/Q693374,2011,769,dcid:Count_Person_Employed_Male, -wikidataId/Q693374,2011,1258,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q693374,2011,636,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q693374,2011,624,dcid:Count_Person_FullTimeEmployee_Male, -wikidataId/Q72,2012,40362,dcid:Count_Company, -wikidataId/Q72,2012,448474,dcid:Count_Person_Employed, -wikidataId/Q72,2012,205879,dcid:Count_Person_Employed_Female, -wikidataId/Q72,2012,242595,dcid:Count_Person_Employed_Male, -wikidataId/Q72,2012,343562,dcid:Count_Person_FullTimeEmployee, -wikidataId/Q72,2012,138749,dcid:Count_Person_FullTimeEmployee_Female, -wikidataId/Q72,2012,204816,dcid:Count_Person_FullTimeEmployee_Male, +observationAbout,observationDate,value,variableMeasured +wikidataId/Q72,2011,39747,dcid:Count_Company +wikidataId/Q72,2011,443621,dcid:Count_Person_Employed +wikidataId/Q72,2011,202970,dcid:Count_Person_Employed_Female +wikidataId/Q72,2011,240651,dcid:Count_Person_Employed_Male +wikidataId/Q72,2011,339750,dcid:Count_Person_FullTimeEmployee +wikidataId/Q72,2011,136406,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q72,2011,203349,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q445559,2011,5249,dcid:Count_Company +wikidataId/Q445559,2011,68627,dcid:Count_Person_Employed +wikidataId/Q445559,2011,34662,dcid:Count_Person_Employed_Female +wikidataId/Q445559,2011,33965,dcid:Count_Person_Employed_Male +wikidataId/Q445559,2011,53138,dcid:Count_Person_FullTimeEmployee +wikidataId/Q445559,2011,24338,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q445559,2011,28799,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q692511,2011,1522,dcid:Count_Company +wikidataId/Q692511,2011,10180,dcid:Count_Person_Employed +wikidataId/Q692511,2011,5492,dcid:Count_Person_Employed_Female +wikidataId/Q692511,2011,4688,dcid:Count_Person_Employed_Male +wikidataId/Q692511,2011,7205,dcid:Count_Person_FullTimeEmployee +wikidataId/Q692511,2011,3493,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q692511,2011,3710,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q39240,2011,700,dcid:Count_Company +wikidataId/Q39240,2011,13203,dcid:Count_Person_Employed +wikidataId/Q39240,2011,7025,dcid:Count_Person_Employed_Female +wikidataId/Q39240,2011,6178,dcid:Count_Person_Employed_Male +wikidataId/Q39240,2011,9090,dcid:Count_Person_FullTimeEmployee +wikidataId/Q39240,2011,4376,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q39240,2011,4714,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q10987378,2011,1193,dcid:Count_Company +wikidataId/Q10987378,2011,15160,dcid:Count_Person_Employed +wikidataId/Q10987378,2011,7649,dcid:Count_Person_Employed_Female +wikidataId/Q10987378,2011,7511,dcid:Count_Person_Employed_Male +wikidataId/Q10987378,2011,12213,dcid:Count_Person_FullTimeEmployee +wikidataId/Q10987378,2011,5602,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q10987378,2011,6608,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q1093831,2011,1834,dcid:Count_Company +wikidataId/Q1093831,2011,30084,dcid:Count_Person_Employed +wikidataId/Q1093831,2011,14496,dcid:Count_Person_Employed_Female +wikidataId/Q1093831,2011,15588,dcid:Count_Person_Employed_Male +wikidataId/Q1093831,2011,24633,dcid:Count_Person_FullTimeEmployee +wikidataId/Q1093831,2011,10861,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q1093831,2011,13768,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q456153,2011,3690,dcid:Count_Company +wikidataId/Q456153,2011,42295,dcid:Count_Person_Employed +wikidataId/Q456153,2011,19218,dcid:Count_Person_Employed_Female +wikidataId/Q456153,2011,23077,dcid:Count_Person_Employed_Male +wikidataId/Q456153,2011,32625,dcid:Count_Person_FullTimeEmployee +wikidataId/Q456153,2011,12786,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q456153,2011,19844,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q642353,2011,1050,dcid:Count_Company +wikidataId/Q642353,2011,7578,dcid:Count_Person_Employed +wikidataId/Q642353,2011,3396,dcid:Count_Person_Employed_Female +wikidataId/Q642353,2011,4182,dcid:Count_Person_Employed_Male +wikidataId/Q642353,2011,5865,dcid:Count_Person_FullTimeEmployee +wikidataId/Q642353,2011,2271,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q642353,2011,3594,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q691367,2011,177,dcid:Count_Company +wikidataId/Q691367,2011,640,dcid:Count_Person_Employed +wikidataId/Q691367,2011,340,dcid:Count_Person_Employed_Female +wikidataId/Q691367,2011,300,dcid:Count_Person_Employed_Male +wikidataId/Q691367,2011,460,dcid:Count_Person_FullTimeEmployee +wikidataId/Q691367,2011,217,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q691367,2011,246,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q648218,2011,2463,dcid:Count_Company +wikidataId/Q648218,2011,34077,dcid:Count_Person_Employed +wikidataId/Q648218,2011,15482,dcid:Count_Person_Employed_Female +wikidataId/Q648218,2011,18595,dcid:Count_Person_Employed_Male +wikidataId/Q648218,2011,26297,dcid:Count_Person_FullTimeEmployee +wikidataId/Q648218,2011,10298,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q648218,2011,16002,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q675017,2011,3716,dcid:Count_Company +wikidataId/Q675017,2011,41539,dcid:Count_Person_Employed +wikidataId/Q675017,2011,18494,dcid:Count_Person_Employed_Female +wikidataId/Q675017,2011,23045,dcid:Count_Person_Employed_Male +wikidataId/Q675017,2011,32146,dcid:Count_Person_FullTimeEmployee +wikidataId/Q675017,2011,12425,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q675017,2011,19724,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q433012,2011,1668,dcid:Count_Company +wikidataId/Q433012,2011,25737,dcid:Count_Person_Employed +wikidataId/Q433012,2011,10281,dcid:Count_Person_Employed_Female +wikidataId/Q433012,2011,15456,dcid:Count_Person_Employed_Male +wikidataId/Q433012,2011,20428,dcid:Count_Person_FullTimeEmployee +wikidataId/Q433012,2011,6962,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q433012,2011,13465,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q693357,2011,358,dcid:Count_Company +wikidataId/Q693357,2011,4510,dcid:Count_Person_Employed +wikidataId/Q693357,2011,3113,dcid:Count_Person_Employed_Female +wikidataId/Q693357,2011,1397,dcid:Count_Person_Employed_Male +wikidataId/Q693357,2011,3523,dcid:Count_Person_FullTimeEmployee +wikidataId/Q693357,2011,2311,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q693357,2011,1209,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q370104,2011,1690,dcid:Count_Company +wikidataId/Q370104,2011,11292,dcid:Count_Person_Employed +wikidataId/Q370104,2011,5100,dcid:Count_Person_Employed_Female +wikidataId/Q370104,2011,6192,dcid:Count_Person_Employed_Male +wikidataId/Q370104,2011,8196,dcid:Count_Person_FullTimeEmployee +wikidataId/Q370104,2011,3148,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q370104,2011,5049,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q677133,2011,3892,dcid:Count_Company +wikidataId/Q677133,2011,36061,dcid:Count_Person_Employed +wikidataId/Q677133,2011,15217,dcid:Count_Person_Employed_Female +wikidataId/Q677133,2011,20844,dcid:Count_Person_Employed_Male +wikidataId/Q677133,2011,27489,dcid:Count_Person_FullTimeEmployee +wikidataId/Q677133,2011,10031,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q677133,2011,17459,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q531899,2011,984,dcid:Count_Company +wikidataId/Q531899,2011,9023,dcid:Count_Person_Employed +wikidataId/Q531899,2011,4202,dcid:Count_Person_Employed_Female +wikidataId/Q531899,2011,4821,dcid:Count_Person_Employed_Male +wikidataId/Q531899,2011,6753,dcid:Count_Person_FullTimeEmployee +wikidataId/Q531899,2011,2834,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q531899,2011,3919,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q1805410,2011,2066,dcid:Count_Company +wikidataId/Q1805410,2011,19957,dcid:Count_Person_Employed +wikidataId/Q1805410,2011,8639,dcid:Count_Person_Employed_Female +wikidataId/Q1805410,2011,11318,dcid:Count_Person_Employed_Male +wikidataId/Q1805410,2011,15238,dcid:Count_Person_FullTimeEmployee +wikidataId/Q1805410,2011,5743,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q1805410,2011,9495,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q870084,2011,842,dcid:Count_Company +wikidataId/Q870084,2011,7081,dcid:Count_Person_Employed +wikidataId/Q870084,2011,2376,dcid:Count_Person_Employed_Female +wikidataId/Q870084,2011,4705,dcid:Count_Person_Employed_Male +wikidataId/Q870084,2011,5497,dcid:Count_Person_FullTimeEmployee +wikidataId/Q870084,2011,1455,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q870084,2011,4045,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q460885,2011,2552,dcid:Count_Company +wikidataId/Q460885,2011,44156,dcid:Count_Person_Employed +wikidataId/Q460885,2011,16755,dcid:Count_Person_Employed_Female +wikidataId/Q460885,2011,27401,dcid:Count_Person_Employed_Male +wikidataId/Q460885,2011,35887,dcid:Count_Person_FullTimeEmployee +wikidataId/Q460885,2011,11991,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q460885,2011,23902,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q693413,2011,1363,dcid:Count_Company +wikidataId/Q693413,2011,15655,dcid:Count_Person_Employed +wikidataId/Q693413,2011,7201,dcid:Count_Person_Employed_Female +wikidataId/Q693413,2011,8454,dcid:Count_Person_Employed_Male +wikidataId/Q693413,2011,11596,dcid:Count_Person_FullTimeEmployee +wikidataId/Q693413,2011,4844,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q693413,2011,6753,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q687052,2011,1189,dcid:Count_Company +wikidataId/Q687052,2011,28501,dcid:Count_Person_Employed +wikidataId/Q687052,2011,9554,dcid:Count_Person_Employed_Female +wikidataId/Q687052,2011,18947,dcid:Count_Person_Employed_Male +wikidataId/Q687052,2011,24293,dcid:Count_Person_FullTimeEmployee +wikidataId/Q687052,2011,7144,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q687052,2011,17146,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q456170,2011,2961,dcid:Count_Company +wikidataId/Q456170,2011,25736,dcid:Count_Person_Employed +wikidataId/Q456170,2011,12597,dcid:Count_Person_Employed_Female +wikidataId/Q456170,2011,13139,dcid:Count_Person_Employed_Male +wikidataId/Q456170,2011,18409,dcid:Count_Person_FullTimeEmployee +wikidataId/Q456170,2011,8037,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q456170,2011,10367,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q656446,2011,1931,dcid:Count_Company +wikidataId/Q656446,2011,15235,dcid:Count_Person_Employed +wikidataId/Q656446,2011,7380,dcid:Count_Person_Employed_Female +wikidataId/Q656446,2011,7855,dcid:Count_Person_Employed_Male +wikidataId/Q656446,2011,11198,dcid:Count_Person_FullTimeEmployee +wikidataId/Q656446,2011,4786,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q656446,2011,6413,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q693483,2011,1030,dcid:Count_Company +wikidataId/Q693483,2011,10501,dcid:Count_Person_Employed +wikidataId/Q693483,2011,5217,dcid:Count_Person_Employed_Female +wikidataId/Q693483,2011,5284,dcid:Count_Person_Employed_Male +wikidataId/Q693483,2011,7210,dcid:Count_Person_FullTimeEmployee +wikidataId/Q693483,2011,3253,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q693483,2011,3956,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q456149,2011,3758,dcid:Count_Company +wikidataId/Q456149,2011,31803,dcid:Count_Person_Employed +wikidataId/Q456149,2011,19099,dcid:Count_Person_Employed_Female +wikidataId/Q456149,2011,12704,dcid:Count_Person_Employed_Male +wikidataId/Q456149,2011,23623,dcid:Count_Person_FullTimeEmployee +wikidataId/Q456149,2011,13268,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q456149,2011,10356,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q693269,2011,838,dcid:Count_Company +wikidataId/Q693269,2011,14560,dcid:Count_Person_Employed +wikidataId/Q693269,2011,8961,dcid:Count_Person_Employed_Female +wikidataId/Q693269,2011,5599,dcid:Count_Person_Employed_Male +wikidataId/Q693269,2011,11196,dcid:Count_Person_FullTimeEmployee +wikidataId/Q693269,2011,6607,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q693269,2011,4590,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q693454,2011,1701,dcid:Count_Company +wikidataId/Q693454,2011,11377,dcid:Count_Person_Employed +wikidataId/Q693454,2011,6669,dcid:Count_Person_Employed_Female +wikidataId/Q693454,2011,4708,dcid:Count_Person_Employed_Male +wikidataId/Q693454,2011,8268,dcid:Count_Person_FullTimeEmployee +wikidataId/Q693454,2011,4448,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q693454,2011,3822,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q476940,2011,715,dcid:Count_Company +wikidataId/Q476940,2011,3824,dcid:Count_Person_Employed +wikidataId/Q476940,2011,2212,dcid:Count_Person_Employed_Female +wikidataId/Q476940,2011,1612,dcid:Count_Person_Employed_Male +wikidataId/Q476940,2011,2678,dcid:Count_Person_FullTimeEmployee +wikidataId/Q476940,2011,1383,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q476940,2011,1296,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q392079,2011,504,dcid:Count_Company +wikidataId/Q392079,2011,2042,dcid:Count_Person_Employed +wikidataId/Q392079,2011,1257,dcid:Count_Person_Employed_Female +wikidataId/Q392079,2011,785,dcid:Count_Person_Employed_Male +wikidataId/Q392079,2011,1483,dcid:Count_Person_FullTimeEmployee +wikidataId/Q392079,2011,833,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q392079,2011,647,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q677710,2011,3675,dcid:Count_Company +wikidataId/Q677710,2011,28912,dcid:Count_Person_Employed +wikidataId/Q677710,2011,15351,dcid:Count_Person_Employed_Female +wikidataId/Q677710,2011,13561,dcid:Count_Person_Employed_Male +wikidataId/Q677710,2011,21358,dcid:Count_Person_FullTimeEmployee +wikidataId/Q677710,2011,10330,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q677710,2011,11025,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q692773,2011,1526,dcid:Count_Company +wikidataId/Q692773,2011,9963,dcid:Count_Person_Employed +wikidataId/Q692773,2011,4975,dcid:Count_Person_Employed_Female +wikidataId/Q692773,2011,4988,dcid:Count_Person_Employed_Male +wikidataId/Q692773,2011,7537,dcid:Count_Person_FullTimeEmployee +wikidataId/Q692773,2011,3393,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q692773,2011,4146,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q693397,2011,1503,dcid:Count_Company +wikidataId/Q693397,2011,9161,dcid:Count_Person_Employed +wikidataId/Q693397,2011,4556,dcid:Count_Person_Employed_Female +wikidataId/Q693397,2011,4605,dcid:Count_Person_Employed_Male +wikidataId/Q693397,2011,6891,dcid:Count_Person_FullTimeEmployee +wikidataId/Q693397,2011,3024,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q693397,2011,3869,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q693321,2011,646,dcid:Count_Company +wikidataId/Q693321,2011,9788,dcid:Count_Person_Employed +wikidataId/Q693321,2011,5820,dcid:Count_Person_Employed_Female +wikidataId/Q693321,2011,3968,dcid:Count_Person_Employed_Male +wikidataId/Q693321,2011,6925,dcid:Count_Person_FullTimeEmployee +wikidataId/Q693321,2011,3913,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q693321,2011,3011,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q456159,2011,3277,dcid:Count_Company +wikidataId/Q456159,2011,48818,dcid:Count_Person_Employed +wikidataId/Q456159,2011,18683,dcid:Count_Person_Employed_Female +wikidataId/Q456159,2011,30135,dcid:Count_Person_Employed_Male +wikidataId/Q456159,2011,38569,dcid:Count_Person_FullTimeEmployee +wikidataId/Q456159,2011,12314,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q456159,2011,26256,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q80797,2011,1024,dcid:Count_Company +wikidataId/Q80797,2011,7797,dcid:Count_Person_Employed +wikidataId/Q80797,2011,3310,dcid:Count_Person_Employed_Female +wikidataId/Q80797,2011,4487,dcid:Count_Person_Employed_Male +wikidataId/Q80797,2011,6040,dcid:Count_Person_FullTimeEmployee +wikidataId/Q80797,2011,2102,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q80797,2011,3940,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q445711,2011,2253,dcid:Count_Company +wikidataId/Q445711,2011,41021,dcid:Count_Person_Employed +wikidataId/Q445711,2011,15373,dcid:Count_Person_Employed_Female +wikidataId/Q445711,2011,25648,dcid:Count_Person_Employed_Male +wikidataId/Q445711,2011,32530,dcid:Count_Person_FullTimeEmployee +wikidataId/Q445711,2011,10215,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q445711,2011,22315,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q456175,2011,1983,dcid:Count_Company +wikidataId/Q456175,2011,15272,dcid:Count_Person_Employed +wikidataId/Q456175,2011,7707,dcid:Count_Person_Employed_Female +wikidataId/Q456175,2011,7565,dcid:Count_Person_Employed_Male +wikidataId/Q456175,2011,10589,dcid:Count_Person_FullTimeEmployee +wikidataId/Q456175,2011,4753,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q456175,2011,5836,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q455496,2011,1031,dcid:Count_Company +wikidataId/Q455496,2011,8233,dcid:Count_Person_Employed +wikidataId/Q455496,2011,3643,dcid:Count_Person_Employed_Female +wikidataId/Q455496,2011,4590,dcid:Count_Person_Employed_Male +wikidataId/Q455496,2011,5796,dcid:Count_Person_FullTimeEmployee +wikidataId/Q455496,2011,2281,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q455496,2011,3515,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q678030,2011,952,dcid:Count_Company +wikidataId/Q678030,2011,7039,dcid:Count_Person_Employed +wikidataId/Q678030,2011,4064,dcid:Count_Person_Employed_Female +wikidataId/Q678030,2011,2975,dcid:Count_Person_Employed_Male +wikidataId/Q678030,2011,4795,dcid:Count_Person_FullTimeEmployee +wikidataId/Q678030,2011,2475,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q678030,2011,2322,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q870098,2011,3902,dcid:Count_Company +wikidataId/Q870098,2011,52782,dcid:Count_Person_Employed +wikidataId/Q870098,2011,21371,dcid:Count_Person_Employed_Female +wikidataId/Q870098,2011,31411,dcid:Count_Person_Employed_Male +wikidataId/Q870098,2011,40609,dcid:Count_Person_FullTimeEmployee +wikidataId/Q870098,2011,13940,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q870098,2011,26668,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q382903,2011,800,dcid:Count_Company +wikidataId/Q382903,2011,3553,dcid:Count_Person_Employed +wikidataId/Q382903,2011,1675,dcid:Count_Person_Employed_Female +wikidataId/Q382903,2011,1878,dcid:Count_Person_Employed_Male +wikidataId/Q382903,2011,2671,dcid:Count_Person_FullTimeEmployee +wikidataId/Q382903,2011,1084,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q382903,2011,1586,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q167179,2011,1767,dcid:Count_Company +wikidataId/Q167179,2011,23926,dcid:Count_Person_Employed +wikidataId/Q167179,2011,10162,dcid:Count_Person_Employed_Female +wikidataId/Q167179,2011,13764,dcid:Count_Person_Employed_Male +wikidataId/Q167179,2011,18381,dcid:Count_Person_FullTimeEmployee +wikidataId/Q167179,2011,6569,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q167179,2011,11812,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q276792,2011,1335,dcid:Count_Company +wikidataId/Q276792,2011,25303,dcid:Count_Person_Employed +wikidataId/Q276792,2011,9534,dcid:Count_Person_Employed_Female +wikidataId/Q276792,2011,15769,dcid:Count_Person_Employed_Male +wikidataId/Q276792,2011,19553,dcid:Count_Person_FullTimeEmployee +wikidataId/Q276792,2011,6288,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q276792,2011,13267,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q657525,2011,945,dcid:Count_Company +wikidataId/Q657525,2011,6868,dcid:Count_Person_Employed +wikidataId/Q657525,2011,3415,dcid:Count_Person_Employed_Female +wikidataId/Q657525,2011,3453,dcid:Count_Person_Employed_Male +wikidataId/Q657525,2011,4762,dcid:Count_Person_FullTimeEmployee +wikidataId/Q657525,2011,1932,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q657525,2011,2828,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q652455,2011,183,dcid:Count_Company +wikidataId/Q652455,2011,1516,dcid:Count_Person_Employed +wikidataId/Q652455,2011,493,dcid:Count_Person_Employed_Female +wikidataId/Q652455,2011,1023,dcid:Count_Person_Employed_Male +wikidataId/Q652455,2011,1197,dcid:Count_Person_FullTimeEmployee +wikidataId/Q652455,2011,319,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q652455,2011,879,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q692728,2011,464,dcid:Count_Company +wikidataId/Q692728,2011,3690,dcid:Count_Person_Employed +wikidataId/Q692728,2011,2029,dcid:Count_Person_Employed_Female +wikidataId/Q692728,2011,1661,dcid:Count_Person_Employed_Male +wikidataId/Q692728,2011,2305,dcid:Count_Person_FullTimeEmployee +wikidataId/Q692728,2011,982,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q692728,2011,1325,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q693374,2011,298,dcid:Count_Company +wikidataId/Q693374,2011,1662,dcid:Count_Person_Employed +wikidataId/Q693374,2011,893,dcid:Count_Person_Employed_Female +wikidataId/Q693374,2011,769,dcid:Count_Person_Employed_Male +wikidataId/Q693374,2011,1258,dcid:Count_Person_FullTimeEmployee +wikidataId/Q693374,2011,636,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q693374,2011,624,dcid:Count_Person_FullTimeEmployee_Male +wikidataId/Q72,2012,40362,dcid:Count_Company +wikidataId/Q72,2012,448474,dcid:Count_Person_Employed +wikidataId/Q72,2012,205879,dcid:Count_Person_Employed_Female +wikidataId/Q72,2012,242595,dcid:Count_Person_Employed_Male +wikidataId/Q72,2012,343562,dcid:Count_Person_FullTimeEmployee +wikidataId/Q72,2012,138749,dcid:Count_Person_FullTimeEmployee_Female +wikidataId/Q72,2012,204816,dcid:Count_Person_FullTimeEmployee_Male diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.tmcf b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.tmcf index 9b9c3598f7..d25521a15c 100644 --- a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.tmcf +++ b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output.tmcf @@ -3,5 +3,4 @@ observationAbout: C:zurich_population_wir_2552_wiki_output->observationAbout observationDate: C:zurich_population_wir_2552_wiki_output->observationDate value: C:zurich_population_wir_2552_wiki_output->value variableMeasured: C:zurich_population_wir_2552_wiki_output->variableMeasured -unit: C:zurich_population_wir_2552_wiki_output->unit typeOf: dcs:StatVarObservation diff --git a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output_stat_vars.mcf b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output_stat_vars.mcf index 818487edb0..9de7b06fbc 100644 --- a/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output_stat_vars.mcf +++ b/statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output_stat_vars.mcf @@ -1,4 +1,11 @@ -# Auto generated using command: "tools/statvar_importer/stat_var_processor.py --input_data=statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv --pv_map=statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv --config_file=statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv --output_path=statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output" on 2026-08-05 11:36:15.561091 +# Auto generated using command: "tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv --pv_map=statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv --config_file=statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv --output_columns=observationAbout,observationDate,value,variableMeasured --output_path=statvar_imports/zurich/wir_2552_wiki/test_data/zurich_population_wir_2552_wiki_output" on 2026-09-03 13:07:07.293325 + +Node: dcid:Count_Company +typeOf: dcid:StatisticalVariable +name: "Count of Company" +populationType: dcid:Company +measuredProperty: dcid:count +statType: dcid:measuredValue Node: dcid:Count_Person_Employed typeOf: dcid:StatisticalVariable @@ -23,3 +30,29 @@ statType: dcid:measuredValue employmentStatus: dcid:Employed gender: dcid:Male +Node: dcid:Count_Person_FullTimeEmployee +typeOf: dcid:StatisticalVariable +name: "Population: Full Time Employee" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +employmentStatus: dcid:FullTimeEmployee + +Node: dcid:Count_Person_FullTimeEmployee_Female +typeOf: dcid:StatisticalVariable +name: "Population: Full Time Employee, Female" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +employmentStatus: dcid:FullTimeEmployee +gender: dcid:Female + +Node: dcid:Count_Person_FullTimeEmployee_Male +typeOf: dcid:StatisticalVariable +name: "Population: Full Time Employee, Male" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +employmentStatus: dcid:FullTimeEmployee +gender: dcid:Male + diff --git a/statvar_imports/zurich/wir_2552_wiki/validation_config.json b/statvar_imports/zurich/wir_2552_wiki/validation_config.json index 66014b3369..59613cd754 100644 --- a/statvar_imports/zurich/wir_2552_wiki/validation_config.json +++ b/statvar_imports/zurich/wir_2552_wiki/validation_config.json @@ -8,6 +8,21 @@ "params": { "threshold": 0.1 } + }, + { + "rule_id": "check_max_date_consistent", + "description": "Checks if the MaxDate is the same for all StatVars.", + "validator": "MAX_DATE_CONSISTENT", + "params": {} + }, + { + "rule_id": "check_max_date_freshness", + "description": "Verifies that MaxDate meets the expected freshness threshold for the dataset.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT MAX(MaxDate) >= '2022' AS is_fresh, COUNT(*) AS row_cnt FROM stats", + "condition": "is_fresh = TRUE AND row_cnt > 0" + } } ] } From 50747a59e74d8105d6dd819110d4f187d24643d9 Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Fri, 4 Sep 2026 10:21:45 +0000 Subject: [PATCH 5/6] Preserve traceback with exc_info in generate_rollups and add main entrypoint tests - Updated main() in generate_rollups.py to log full exception traceback via logging.fatal(..., exc_info=True). - Added unit tests for main() verifying successful execution and logging behavior on exception. BUG=542856409,542859819 TAG=agy CONV=e14d0b1e-8efd-44a0-b3d1-5640746680a4 --- .../zurich/wir_2552_wiki/generate_rollups.py | 5 +++- .../wir_2552_wiki/generate_rollups_test.py | 28 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py b/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py index 84f4625ca9..975a3b6614 100644 --- a/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py +++ b/statvar_imports/zurich/wir_2552_wiki/generate_rollups.py @@ -125,7 +125,10 @@ def generate_rollups(input_csv: str, output_csv: str) -> pd.DataFrame: def main(argv): del argv # Unused. - generate_rollups(FLAGS.input_csv, FLAGS.output_csv) + try: + generate_rollups(FLAGS.input_csv, FLAGS.output_csv) + except Exception as e: + logging.fatal("Failed to generate rollups: %s", e, exc_info=True) if __name__ == '__main__': diff --git a/statvar_imports/zurich/wir_2552_wiki/generate_rollups_test.py b/statvar_imports/zurich/wir_2552_wiki/generate_rollups_test.py index a2c5c321b9..9d601374e3 100644 --- a/statvar_imports/zurich/wir_2552_wiki/generate_rollups_test.py +++ b/statvar_imports/zurich/wir_2552_wiki/generate_rollups_test.py @@ -17,6 +17,7 @@ import sys import tempfile import unittest +from unittest.mock import patch import numpy as np import pandas as pd @@ -25,7 +26,7 @@ if _MODULE_DIR not in sys.path: sys.path.insert(0, _MODULE_DIR) -from generate_rollups import generate_rollups, process_rollups +from generate_rollups import FLAGS, generate_rollups, main, process_rollups class GenerateRollupsTest(unittest.TestCase): @@ -33,6 +34,7 @@ class GenerateRollupsTest(unittest.TestCase): def setUp(self): super().setUp() + FLAGS.mark_as_parsed() self.sample_raw_data = pd.DataFrame({ 'Jahr': [2020, 2020, 2020, 2020], 'RaumSort': [1, 1, 1, 1], @@ -152,6 +154,30 @@ def test_process_rollups_string_sort_columns(self): self.assertEqual(len(result_df), 1) self.assertEqual(result_df.iloc[0]['Arbeitsstaetten'], 100.0) + def test_main_success(self): + """Verifies that main executes successfully with valid flag parameters.""" + with tempfile.TemporaryDirectory() as tmp_dir: + input_csv = os.path.join(tmp_dir, 'input.csv') + output_csv = os.path.join(tmp_dir, 'output.csv') + self.sample_raw_data.to_csv(input_csv, + index=False, + encoding='utf-8') + FLAGS.input_csv = input_csv + FLAGS.output_csv = output_csv + main([]) + self.assertTrue(os.path.exists(output_csv)) + + def test_main_failure_logs_fatal(self): + """Verifies that main logs a fatal error with exc_info on exception.""" + FLAGS.input_csv = '/non/existent/path/file.csv' + FLAGS.output_csv = '/tmp/dummy_output.csv' + with patch('generate_rollups.logging.fatal') as mock_fatal: + main([]) + mock_fatal.assert_called_once() + self.assertIn('Failed to generate rollups', + mock_fatal.call_args[0][0]) + self.assertTrue(mock_fatal.call_args[1].get('exc_info')) + if __name__ == '__main__': unittest.main() From 3d3fc4ff0d2a62a03b13b23bb1fdfdf59a0c5c19 Mon Sep 17 00:00:00 2001 From: Abhishek Jaiswal Date: Fri, 4 Sep 2026 11:21:14 +0000 Subject: [PATCH 6/6] Revert .gitignore modification TAG=agy CONV=e14d0b1e-8efd-44a0-b3d1-5640746680a4 --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index f898268d3c..87f3d76b24 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,4 @@ import-automation/executor/config_override.json # Ignore generated troubleshooting post-mortem documents agents/troubleshooting/ -# Local past conversation shortcuts -past_conversations/ -