Skip to content

Various fixes and updates #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SanderNielen wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Various fixes and updates #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
224 changes: 192 additions & 32 deletions mysite/dpp/lca.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,19 @@
"""Toolbox with functions for doing LCA calculations,
and to import and export DPP data to Brightway.
"""
import bw2data as bwd
import bw2io as bwi
import datetime
import json
import logging
import time
import requests

from pathlib import Path
import pandas as pd
from django.conf import settings

import bw2data as bwd
import bw2io as bwi
from bw2io.importers.bonsai import BonsaiImporter

logger = logging.getLogger(__name__)

Expand All @@ -25,15 +34,110 @@
"human toxicity: non-carcinogenic, inorganics', 'comparative toxic unit for human (CTUh)",
"human toxicity: non-carcinogenic, organics', 'comparative toxic unit for human (CTUh)')",
}
DEFAULT_REMOTE_PROJECT = "ecoinvent-3.12-biosphere"
DPP_PROJECT = "L4M-DPP"
BIOSPHERE = "ecoinvent-3.12-biosphere"
ECOINVENT = "ecoinvent-3-12-cutoff"
BONSAI_DB = "BONSAI v2.4.0"

CACHE_DIR = Path.cwd() / "_cache"
BONSAI_EXPORT_DIR = Path.cwd() / "bonsai_data"
CACHE_DIR.mkdir(exist_ok=True)
BONSAI_EXPORT_DIR.mkdir(parents=True, exist_ok=True)

# ---------------------------------------------------------------------------
# Project setup and database import
# ---------------------------------------------------------------------------

country_data = pd.read_csv(settings.DATA_DIR / "country_data.csv", index_col="country")

def setup_project(project_name: str) -> None:
"""Initialize the project if needed, and check that it is complete."""
def project_status(project_name: str) -> dict[str, bool]:
"""Check if all databases in a project are fully processed.

A database counts as True only if its processed file exists on disk
(an import that failed half-way leaves only a registered DB).
Returns {db_name: processed}, or {} if the project does not exist.
"""
if project_name not in bwd.projects:
logger.debug("Creating a new Brightway project setup.")
bwi.remote.install_project(DEFAULT_REMOTE_PROJECT, project_name)
return {}
bwd.projects.set_current(project_name)
return {
name: Path(bwd.Database(name).filepath_processed()).is_file()
for name in bwd.databases
}

def staged_bonsai_export(export_dir: Path|None=None) -> tuple[Path, dict]:
"""
The export staged in the cache with its units fixed, plus the BONSAI -> biosphere map.

v2.4.0 labels standing wood "standard cubic meter", ecoinvent "cubic meter", and the
importer's unit library does not know the former. For solid wood the two are the same
volume, so the label is aligned 1:1. The export itself is not modified.
"""
export_dir = export_dir or BONSAI_EXPORT_DIR
export_files = [
"io_metadata.json", "product_value_table.gzip",
"technosphere_value_table.gzip", "extensions_value_table.gzip",
"index_table_technosphere.gzip", "index_table_extensions.gzip",
]
stage = CACHE_DIR / "bonsai_export"
stage.mkdir(exist_ok=True)
for name in export_files:
link = stage / name
link.unlink(missing_ok=True)
if name != "io_metadata.json":
link.symlink_to((export_dir / name).resolve())

metadata = json.loads((export_dir / "io_metadata.json").read_text(encoding="utf-8"))
biosphere = bwd.Database(bwd.config.biosphere)
to_biosphere = {}
for code, flow in metadata.items():
if "compartment" not in flow or not flow.get("source_id"):
continue # unlinked flows go to "<database> biosphere"
target = biosphere.get(code=flow["source_id"])
to_biosphere[code] = target["code"]
if flow.get("unit") == "standard cubic meter" and target["unit"] == "cubic meter":
print(f" unit aligned 1:1 to 'cubic meter': {flow['name']}")
flow["unit"] = "cubic meter"
(stage / "io_metadata.json").write_text(json.dumps(metadata), encoding="utf-8")
return stage, to_biosphere

def import_bonsai() -> None:
"""Import the BONSAI database using a staged export approach."""
stage, to_biosphere = staged_bonsai_export()
importer = BonsaiImporter(stage, BONSAI_DB, b3mapping=to_biosphere)
importer.apply_strategies()
importer.write_database()

def setup_project(project_name: str=DPP_PROJECT) -> None:
"""Initialize the project if needed, add biosphere and BONSAI
and check that it is complete.
"""
status = project_status(project_name)

if project_name not in bwd.projects:
logger.debug("Creating a new Brightway project setup.")
if BIOSPHERE not in status or not status[BIOSPHERE]:
scratch = f"{project_name}_setup"
bwd.projects.set_current(scratch)
bwi.remote.install_project(project_key=BIOSPHERE,
project_name=project_name, overwrite_existing=True)
bwd.projects.set_current(project_name)
bwd.projects.delete_project(scratch, delete_dir=True)
else:
bwd.projects.set_current(project_name)
if "bonsai" not in status or not status["bonsai"]:
started = time.perf_counter()
logger.info(f" importing {BONSAI_DB} from {BONSAI_EXPORT_DIR} …")
if BONSAI_DB in bwd.databases: # left behind by an import that failed half-way
logger.info(f" removing an incomplete '{BONSAI_DB}'")
del bwd.databases[BONSAI_DB]
import_bonsai()
logger.info(f" done in {time.perf_counter() - started:,.0f} s")

# Check and log the new status
for db, present in project_status(project_name).items():
logger.info(f"{db:<10} {'present' if present else 'MISSING'}")


def ensure_methods(family: str):
"""
Expand Down Expand Up @@ -89,6 +193,32 @@ def prompt_choice(title: str, options: list, default_index: int = 0):
print("Invalid choice, using default.")
return options[default_index]


# ---------------------------------------------------------------------------
# Conversion and communication functions
# ---------------------------------------------------------------------------

def bonsai_api(endpoint: str) -> list:
"""Pages of BONSAI API endpoint (no account needed), cached in CACHE_DIR.
Examples of `endpoint`: 'products/', 'activities/', 'product-corr/'
"""
# Load data from cache if available
path = CACHE_DIR / f"bonsai_api_{endpoint.strip('/')}.json"
if path.exists():
return json.loads(path.read_text(encoding="utf-8"))

rows = []
url = "https://lca.aau.dk/api/" + endpoint
while url:
page = requests.get(url, headers={"Accept": "application/json"}, timeout=60)
page.raise_for_status()
data = page.json()
rows += data["results"]
url = (data.get("next") or "").replace("http://", "https://")
path.write_text(json.dumps(rows), encoding="utf-8")
return rows


def get_or_create_bw_process(dpp_product):
raise NotImplementedError()
return
Expand All @@ -106,8 +236,8 @@ def biosphere_to_dpp_flows():
'square meter-year': 'm2-yr',
'standard cubic meter': 'Nm3',
}
setup_project("L4M-DPP")
biosphere_db = bwd.Database(DEFAULT_REMOTE_PROJECT)
setup_project()
biosphere_db = bwd.Database(BIOSPHERE)
bio_flows = set((i['name'], i['unit']) for i in list(biosphere_db))
if len(bio_flows) <= len(Emission.objects.all()): return
for name, unit in bio_flows:
Expand Down Expand Up @@ -166,6 +296,9 @@ def make_transport_exchange(transports, product, amount):
transports: QuerySet of Transport table
product: ProductModel, input to some process
amount: int, amount of product transported

Returns:
Brightway exchange, describing transport input
"""
# Import here to avoid circular imports
from .models import UNIT_CHOICES, CONVERSIONS
Expand All @@ -178,6 +311,15 @@ def make_transport_exchange(transports, product, amount):
"delivery van": '558949a7ddfcccba760eab39bda68e88', # light commercial vehicle, EU
"NA": 'dd736ab7d965646d04b091ae5fa68d97', #FIXME defaults to truck for now
}
bonsai_codes = {
"train": "M_TLAND_ALL",
"ocean ship": "M_TWATR_ALL",
"truck": "M_TLAND_ALL",
"inland ship": "M_TWATR_ALL",
"airplane": "C_TAIR",
"delivery van": "M_TLAND_ALL",
"NA": "M_TLAND_ALL",
}
transport = transports.filter(product=product).first()
if transport is None:
raise LookupError(f"No transport specified for {product}")
Expand All @@ -192,12 +334,45 @@ def make_transport_exchange(transports, product, amount):
else:
mass = 1

return { # Calculate transport distance (t-km)
"input": (ECOINVENT, ecoinvent_codes[transport.mode]),
"amount": mass / 1000 * amount * transport.distance,
"type": "technosphere",
"unit": 'ton kilometer',
}
# Calculate transport distance (t-km) or price (MEuro)
if ECOINVENT in bwd.databases:
exchange = {
"input": (ECOINVENT, ecoinvent_codes[transport.mode]),
"amount": mass / 1000 * amount * transport.distance,
"type": "technosphere",
"unit": 'ton kilometer',
}
elif BONSAI_DB in bwd.databases:
transp_code = bonsai_codes[transport.mode]
price = country_data.loc["?", transp_code + "_price"] #FIXME: country
exchange = {
"input": (BONSAI_DB, transp_code),
"amount": mass * amount * transport.distance * price / 1e9,
# units: kg/pcs * pcs * km * EUR/t-km -> MEuro
"type": "technosphere",
"unit": 'MEUR',
}
return exchange

def list_background_processes() -> dict:
"""
Create a dictionary of available background processes
Structured as: {database_name: {activity_name: activity_code}}
"""
setup_project()
background = {}
for db_name in bwd.databases:
if db_name[:4] != "dpp_":
background[db_name] = {
act["name"]: act["code"]
for act in bwd.Database(db_name)
}
return background


# ---------------------------------------------------------------------------
# Create DPP LCA and do LCA calculations
# ---------------------------------------------------------------------------

def convert_dpp_to_brightway(processes: list, db_name: str):
"""
Expand All @@ -209,7 +384,7 @@ def convert_dpp_to_brightway(processes: list, db_name: str):
# Import here to avoid circular imports
from .models import ProductExchange, EnvExchange, BackgroundProcess

biosphere = bwd.Database(DEFAULT_REMOTE_PROJECT)
biosphere = bwd.Database(BIOSPHERE)
bw_activities = {}
for dpp_process in processes:
# Skip already known background processes
Expand Down Expand Up @@ -339,7 +514,7 @@ def create_supply_chain_lca(product):
# Import here to avoid circular imports
from .models import SustainabilityEvaluation, SustainabilityScore, ImpactIndicator

setup_project("L4M-DPP")
setup_project()
lcia_family = 'EF v3.1'
method_set = ensure_methods(lcia_family)
evaluation, created = SustainabilityEvaluation.objects.get_or_create(
Expand Down Expand Up @@ -411,18 +586,3 @@ def create_supply_chain_lca(product):
)

return evaluation

def list_background_processes() -> dict:
"""
Create a dictionary of available background processes
Structured as: {database_name: {activity_name: activity_code}}
"""
setup_project("L4M-DPP")
background = {}
for db_name in bwd.databases:
if db_name[:4] != "dpp_":
background[db_name] = {
act["name"]: act["code"]
for act in bwd.Database(db_name)
}
return background
Loading
Loading