[Zurich] Update Zurich_Population and Zurich_Population_Number_Of_Company_Workplace_Employees

statvar_imports/zurich/README.md

@@ -1,29 +1,31 @@
 1. import_name": "Zurich_Population_Number_Of_Company_Workplace_Employees"
 
 2. Import Overview
-Zurich population data at Province and City Level.
-Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/wir_2552_wiki.csv)
+Number of companies, workplaces and employees in Zurich city at City, District (Kreise), and Quarter (Quartiere) Level.
+Source URL: [BFS WIR STATENT Data](https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552)
 Import Type: Fully Autorefresh
 Source Data Availability: 2011 to 2022
 Release Frequency: P1Y
 
-3. Preprocessing Steps (No)
+3. Preprocessing Steps (Yes)
+Generate rollups from the downloaded dataset:
+python3 wir_2552_wiki/generate_rollups.py
 
 4. Autorefresh Type
 
 Fully Autorefresh:"0 2 29 * * " (Runs at 2:00 AM on the 29th day of every month).
 
 5. Script Execution Details
 
-" python3 stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data='../../statvar_imports/zurich/wir_2552_wiki/test_data/wir_2552_wiki_input.csv' --pv_map='../../statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_pvmap.csv' --config_file='../../statvar_imports/zurich/wir_2552_wiki/wir_2552_wiki_metadata.csv' --output_path=../../statvar_imports/zurich/wir_2552_wiki/output/zurich_population_wir_2552_wiki_output  "
+" python3 ../../util/download_util_script.py --download_url=https://data.stadt-zuerich.ch/dataset/bfs_wir_statent_ast_beschaeftigte_vza_rechtsform_betrgr_jahr_od2552/download/WIR255OD2552.csv --output_folder=wir_2552_wiki/input_files && python3 wir_2552_wiki/generate_rollups.py && python3 ../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=wir_2552_wiki/input_files/WIR255OD2552_rollups.csv --pv_map=wir_2552_wiki/wir_2552_wiki_pvmap.csv --config_file=wir_2552_wiki/wir_2552_wiki_metadata.csv --output_columns=observationAbout,observationDate,value,variableMeasured --output_path=wir_2552_wiki/output/zurich_population_wir_2552_wiki --output_counters=wir_2552_wiki/counters/zurich_population_wir_2552_wiki_counters.csv "
 
 #####
 
 
 1. import_name": "Zurich_Population_By_Age"
 
 2. Import Overview
-Zurich population data at Province and City Level.
+Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level.
 Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3903_age10_wiki.csv)
 Import Type: Fully Autorefresh
 Source Data Availability: 1993 to 2023
@@ -45,8 +47,8 @@ Fully Autorefresh:" 0 6 29 * * " (Runs at 6:00 AM on the 29th of every month).
 1. import_name": "Zurich_Population"
 
 2. Import Overview
-Zurich population data at Province and City Level.
-Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3240_wiki.csv)
+Total population of Zurich city by quarter and year at City, District (Kreise), and Quarter (Quartiere) Level.
+Source URL: [BEV324OD3240 Dataset](https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240)
 Import Type: Fully Autorefresh
 Source Data Availability: 1941 to 2023
 Release Frequency: P1Y
@@ -59,15 +61,15 @@ Fully Autorefresh:" 30 11 29 * * " (Runs at 11:30 AM on the 29th of every month)
 
 5. Script Execution Details
 
-" python3 stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data='../../statvar_imports/zurich/bev_3240_wiki/test_data/bev_3240_wiki_input.csv' --pv_map='../../statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_pvmap.csv' --config_file='../../statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv' --output_path=../../statvar_imports/zurich/bev_3240_wiki/output/zurich_population_bev_3240_wiki.csv_output "
+" python3 ../../util/download_util_script.py --download_url=https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240/download/BEV324OD3240.csv --output_folder=bev_3240_wiki/input_files && python3 ../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=bev_3240_wiki/input_files/BEV324OD3240.csv --pv_map=bev_3240_wiki/bev_3240_wiki_pvmap.csv --config_file=bev_3240_wiki/bev_3240_wiki_metadata.csv --output_columns=observationAbout,observationDate,value,variableMeasured --output_path=bev_3240_wiki/output/zurich_population_bev_3240_wiki --output_counters=bev_3240_wiki/counters/zurich_population_bev_3240_wiki_counters.csv "
 
 #####
 
 
 1. import_name": "Zurich_Population_Number_Of_Birth_By_Origin"
 
 2. Import Overview
-Zurich population data at Province and City Level.
+Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level.
 Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_4031_hel_wiki.csv)
 Import Type: Fully Autorefresh
 Source Data Availability: 1998 to 2023
@@ -89,7 +91,7 @@ Fully Autorefresh:" 45 15 29 * * " (Runs at 3:45 PM on the 29th of every month).
 1. import_name": "Zurich_Population_Number_Of_Birth"
 
 2. Import Overview
-Zurich population data at Province and City Level.
+Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level.
 Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_4031_wiki.csv)
 Import Type: Fully Autorefresh
 Source Data Availability: 1998 to 2023
@@ -111,7 +113,7 @@ Fully Autorefresh:" 0 20 29 * * " (Runs at 8:00 PM on the 29th of every month).
 1. import_name": "Zurich_Population_By_Origin"
 
 2. Import Overview
-Zurich population data at Province and City Level.
+Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level.
 Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3903_hel_wiki.csv)
 Import Type: Fully Autorefresh
 Source Data Availability: 1993 to 2023
@@ -134,7 +136,7 @@ Fully Autorefresh:" 15 1 29 * * " (Runs at 1:15 AM on the 29th of every month).
 1. import_name": "Zurich_Population_Number_Of_Birth_By_Sex"
 
 2. Import Overview
-Zurich population data at Province and City Level.
+Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level.
 Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_4031_sex_wiki.csv)
 Import Type: Fully Autorefresh
 Source Data Availability: 1993 to 2023
@@ -156,7 +158,7 @@ Fully Autorefresh:" 07 19 29 * * " (Runs at 7:07 PM on the 29th of every month).
 1. import_name": "Zurich_Population_By_Sex"
 
 2. Import Overview
-Zurich population data at Province and City Level.
+Zurich population data at City, District (Kreise), and Quarter (Quartiere) Level.
 Source URL: [Zurich country website](https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3903_sex_wiki.csv)
 Import Type: Fully Autorefresh
 Source Data Availability: 1993 to 2023

statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_metadata.csv

@@ -1,14 +1,14 @@
 parameter,value
-url,https://www.stadt-zuerich.ch/content/dam/web/de/politik-verwaltung/statistik-und-daten/linked-open-data/datacommons/bev_3240_wiki.csv
+url,https://data.stadt-zuerich.ch/dataset/bev_bestand_jahr_quartier_od3240
 description,
-place_type,Country
+place_type,City
 #places_within,
 start_date,
 end_date,
 release_frequency,
 process,
 comments,
-output_columns,"observationAbout,observationDate,value,variableMeasured,unit"
+output_columns,"observationAbout,observationDate,value,variableMeasured"
 header_rows,1
-#input_rows,10
-#Import_Name,Zurich_Population
+reconcile_nodes,False
+output_counters,bev_3240_wiki/counters/zurich_population_bev_3240_wiki_counters.csv

statvar_imports/zurich/bev_3240_wiki/bev_3240_wiki_pvmap.csv

@@ -1,4 +1,37 @@
-key,prop,val,p1,v1
-StichtagDatJahr,observationDate,{Number},,
-AnzBestWir,value,{Number},populationType,Person
-wikiId,observationAbout,wikidataId/{Data},,
+key,prop,val,p1,v1
+StichtagDatJahr,observationDate,{Number},,
+AnzBestWir,value,{Number},populationType,Person
+QuarLang:Rathaus,observationAbout,wikidataId/Q692511,,
+QuarLang:Hochschulen,observationAbout,wikidataId/Q39240,,
+QuarLang:Lindenhof,observationAbout,wikidataId/Q10987378,,
+QuarLang:City,observationAbout,wikidataId/Q1093831,,
+QuarLang:Wollishofen,observationAbout,wikidataId/Q642353,,
+QuarLang:Leimbach,observationAbout,wikidataId/Q691367,,
+QuarLang:Enge,observationAbout,wikidataId/Q648218,,
+QuarLang:Alt-Wiedikon,observationAbout,wikidataId/Q433012,,
+QuarLang:Friesenberg,observationAbout,wikidataId/Q693357,,
+QuarLang:Sihlfeld,observationAbout,wikidataId/Q370104,,
+QuarLang:Werd,observationAbout,wikidataId/Q531899,,
+QuarLang:Langstrasse,observationAbout,wikidataId/Q1805410,,
+QuarLang:Hard,observationAbout,wikidataId/Q870084,,
+QuarLang:Gewerbeschule,observationAbout,wikidataId/Q693413,,
+QuarLang:Escher Wyss,observationAbout,wikidataId/Q687052,,
+QuarLang:Unterstrass,observationAbout,wikidataId/Q656446,,
+QuarLang:Oberstrass,observationAbout,wikidataId/Q693483,,
+QuarLang:Fluntern,observationAbout,wikidataId/Q693269,,
+QuarLang:Hottingen,observationAbout,wikidataId/Q693454,,
+QuarLang:Hirslanden,observationAbout,wikidataId/Q476940,,
+QuarLang:Witikon,observationAbout,wikidataId/Q392079,,
+QuarLang:Seefeld,observationAbout,wikidataId/Q692773,,
+QuarLang:Mühlebach,observationAbout,wikidataId/Q693397,,
+QuarLang:Weinegg,observationAbout,wikidataId/Q693321,,
+QuarLang:Albisrieden,observationAbout,wikidataId/Q80797,,
+QuarLang:Altstetten,observationAbout,wikidataId/Q445711,,
+QuarLang:Höngg,observationAbout,wikidataId/Q455496,,
+QuarLang:Wipkingen,observationAbout,wikidataId/Q678030,,
+QuarLang:Affoltern,observationAbout,wikidataId/Q382903,,
+QuarLang:Oerlikon,observationAbout,wikidataId/Q167179,,
+QuarLang:Seebach,observationAbout,wikidataId/Q276792,,
+QuarLang:Saatlen,observationAbout,wikidataId/Q652455,,
+QuarLang:Schwamendingen-Mitte,observationAbout,wikidataId/Q692728,,
+QuarLang:Hirzenbach,observationAbout,wikidataId/Q693374,,

statvar_imports/zurich/bev_3240_wiki/test_data/bev_3240_wiki_input.csv

@@ -1,25 +1,51 @@
-StichtagDatJahr,AnzBestWir,wikiId
-2023,3343,Q692511
-2022,3330,Q692511
-2021,3275,Q692511
-2020,3317,Q692511
-2019,3307,Q692511
-2018,3267,Q692511
-2017,3266,Q692511
-2016,3226,Q692511
-2015,3197,Q692511
-2014,3236,Q692511
-2013,3194,Q692511
-2012,3164,Q692511
-2011,3140,Q692511
-2010,3099,Q692511
-2009,3110,Q692511
-2008,3051,Q692511
-2007,3079,Q692511
-2006,3097,Q692511
-2005,3081,Q692511
-2004,3115,Q692511
-2003,3149,Q692511
-2002,3128,Q692511
-2001,3103,Q692511
-2000,3109,Q692511
+StichtagDatJahr,QuarSort,QuarLang,AnzBestWir
+2025,11,Rathaus,3252
+2024,11,Rathaus,3260
+2023,11,Rathaus,3343
+2022,11,Rathaus,3330
+2021,11,Rathaus,3275
+2020,11,Rathaus,3317
+2019,11,Rathaus,3307
+2018,11,Rathaus,3267
+2017,11,Rathaus,3266
+2016,11,Rathaus,3226
+2015,11,Rathaus,3197
+2014,11,Rathaus,3236
+2013,11,Rathaus,3194
+2012,11,Rathaus,3164
+2011,11,Rathaus,3140
+2010,11,Rathaus,3099
+2009,11,Rathaus,3110
+2008,11,Rathaus,3051
+2007,11,Rathaus,3079
+2006,11,Rathaus,3097
+2005,11,Rathaus,3081
+2004,11,Rathaus,3115
+2003,11,Rathaus,3149
+2002,11,Rathaus,3128
+2001,11,Rathaus,3103
+2000,11,Rathaus,3109
+1999,11,Rathaus,3104
+1998,11,Rathaus,3050
+1997,11,Rathaus,3083
+1996,11,Rathaus,3078
+1995,11,Rathaus,3049
+1994,11,Rathaus,3089
+1993,11,Rathaus,3133
+1992,11,Rathaus,3186
+1991,11,Rathaus,3173
+1990,11,Rathaus,3145
+1989,11,Rathaus,3161
+1988,11,Rathaus,3187
+1987,11,Rathaus,3218
+1986,11,Rathaus,3177
+1985,11,Rathaus,3181
+1984,11,Rathaus,3277
+1983,11,Rathaus,3304
+1982,11,Rathaus,3356
+1981,11,Rathaus,3501
+1980,11,Rathaus,3539
+1979,11,Rathaus,3551
+1978,11,Rathaus,3710
+1977,11,Rathaus,3802
+1976,11,Rathaus,3836

statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.csv

@@ -1,25 +1,51 @@
-observationAbout,observationDate,value,variableMeasured,unit
-wikidataId/Q692511,2023,3343,dcid:Count_Person,
-wikidataId/Q692511,2022,3330,dcid:Count_Person,
-wikidataId/Q692511,2021,3275,dcid:Count_Person,
-wikidataId/Q692511,2020,3317,dcid:Count_Person,
-wikidataId/Q692511,2019,3307,dcid:Count_Person,
-wikidataId/Q692511,2018,3267,dcid:Count_Person,
-wikidataId/Q692511,2017,3266,dcid:Count_Person,
-wikidataId/Q692511,2016,3226,dcid:Count_Person,
-wikidataId/Q692511,2015,3197,dcid:Count_Person,
-wikidataId/Q692511,2014,3236,dcid:Count_Person,
-wikidataId/Q692511,2013,3194,dcid:Count_Person,
-wikidataId/Q692511,2012,3164,dcid:Count_Person,
-wikidataId/Q692511,2011,3140,dcid:Count_Person,
-wikidataId/Q692511,2010,3099,dcid:Count_Person,
-wikidataId/Q692511,2009,3110,dcid:Count_Person,
-wikidataId/Q692511,2008,3051,dcid:Count_Person,
-wikidataId/Q692511,2007,3079,dcid:Count_Person,
-wikidataId/Q692511,2006,3097,dcid:Count_Person,
-wikidataId/Q692511,2005,3081,dcid:Count_Person,
-wikidataId/Q692511,2004,3115,dcid:Count_Person,
-wikidataId/Q692511,2003,3149,dcid:Count_Person,
-wikidataId/Q692511,2002,3128,dcid:Count_Person,
-wikidataId/Q692511,2001,3103,dcid:Count_Person,
-wikidataId/Q692511,2000,3109,dcid:Count_Person,
+observationAbout,observationDate,value,variableMeasured
+wikidataId/Q692511,2025,3252,dcid:Count_Person
+wikidataId/Q692511,2024,3260,dcid:Count_Person
+wikidataId/Q692511,2023,3343,dcid:Count_Person
+wikidataId/Q692511,2022,3330,dcid:Count_Person
+wikidataId/Q692511,2021,3275,dcid:Count_Person
+wikidataId/Q692511,2020,3317,dcid:Count_Person
+wikidataId/Q692511,2019,3307,dcid:Count_Person
+wikidataId/Q692511,2018,3267,dcid:Count_Person
+wikidataId/Q692511,2017,3266,dcid:Count_Person
+wikidataId/Q692511,2016,3226,dcid:Count_Person
+wikidataId/Q692511,2015,3197,dcid:Count_Person
+wikidataId/Q692511,2014,3236,dcid:Count_Person
+wikidataId/Q692511,2013,3194,dcid:Count_Person
+wikidataId/Q692511,2012,3164,dcid:Count_Person
+wikidataId/Q692511,2011,3140,dcid:Count_Person
+wikidataId/Q692511,2010,3099,dcid:Count_Person
+wikidataId/Q692511,2009,3110,dcid:Count_Person
+wikidataId/Q692511,2008,3051,dcid:Count_Person
+wikidataId/Q692511,2007,3079,dcid:Count_Person
+wikidataId/Q692511,2006,3097,dcid:Count_Person
+wikidataId/Q692511,2005,3081,dcid:Count_Person
+wikidataId/Q692511,2004,3115,dcid:Count_Person
+wikidataId/Q692511,2003,3149,dcid:Count_Person
+wikidataId/Q692511,2002,3128,dcid:Count_Person
+wikidataId/Q692511,2001,3103,dcid:Count_Person
+wikidataId/Q692511,2000,3109,dcid:Count_Person
+wikidataId/Q692511,1999,3104,dcid:Count_Person
+wikidataId/Q692511,1998,3050,dcid:Count_Person
+wikidataId/Q692511,1997,3083,dcid:Count_Person
+wikidataId/Q692511,1996,3078,dcid:Count_Person
+wikidataId/Q692511,1995,3049,dcid:Count_Person
+wikidataId/Q692511,1994,3089,dcid:Count_Person
+wikidataId/Q692511,1993,3133,dcid:Count_Person
+wikidataId/Q692511,1992,3186,dcid:Count_Person
+wikidataId/Q692511,1991,3173,dcid:Count_Person
+wikidataId/Q692511,1990,3145,dcid:Count_Person
+wikidataId/Q692511,1989,3161,dcid:Count_Person
+wikidataId/Q692511,1988,3187,dcid:Count_Person
+wikidataId/Q692511,1987,3218,dcid:Count_Person
+wikidataId/Q692511,1986,3177,dcid:Count_Person
+wikidataId/Q692511,1985,3181,dcid:Count_Person
+wikidataId/Q692511,1984,3277,dcid:Count_Person
+wikidataId/Q692511,1983,3304,dcid:Count_Person
+wikidataId/Q692511,1982,3356,dcid:Count_Person
+wikidataId/Q692511,1981,3501,dcid:Count_Person
+wikidataId/Q692511,1980,3539,dcid:Count_Person
+wikidataId/Q692511,1979,3551,dcid:Count_Person
+wikidataId/Q692511,1978,3710,dcid:Count_Person
+wikidataId/Q692511,1977,3802,dcid:Count_Person
+wikidataId/Q692511,1976,3836,dcid:Count_Person

statvar_imports/zurich/bev_3240_wiki/test_data/zurich_population_bev_3240_wiki_output.tmcf

@@ -1,7 +1,6 @@
-Node: E:bev_3240_wiki_output->E0
-observationAbout: C:bev_3240_wiki_output->observationAbout
-observationDate: C:bev_3240_wiki_output->observationDate
-value: C:bev_3240_wiki_output->value
-variableMeasured: C:bev_3240_wiki_output->variableMeasured
-unit: C:bev_3240_wiki_output->unit
+Node: E:zurich_population_bev_3240_wiki_output->E0
+observationAbout: C:zurich_population_bev_3240_wiki_output->observationAbout
+observationDate: C:zurich_population_bev_3240_wiki_output->observationDate
+value: C:zurich_population_bev_3240_wiki_output->value
+variableMeasured: C:zurich_population_bev_3240_wiki_output->variableMeasured
 typeOf: dcs:StatVarObservation

statvar_imports/zurich/bev_3240_wiki/validation_config.json

@@ -0,0 +1,28 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Checks that the percentage of deleted points is within the threshold.",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 0.1
+            }
+        },
+        {
+            "rule_id": "check_max_date_consistent",
+            "description": "Checks if the MaxDate is the same for all StatVars.",
+            "validator": "MAX_DATE_CONSISTENT",
+            "params": {}
+        },
+        {
+            "rule_id": "check_max_date_freshness",
+            "description": "Verifies that MaxDate meets the expected freshness threshold for the dataset.",
+            "validator": "SQL_VALIDATOR",
+            "params": {
+                "query": "SELECT MAX(MaxDate) >= '2023' AS is_fresh, COUNT(*) AS row_cnt FROM stats",
+                "condition": "is_fresh = TRUE AND row_cnt > 0"
+            }
+        }
+    ]
+}