diff --git a/data.py b/data.py index e053b0c..b8fc345 100644 --- a/data.py +++ b/data.py @@ -2,6 +2,7 @@ import requests import time import json +import gzip from json import JSONEncoder from pathlib import Path from functools import lru_cache @@ -10,9 +11,9 @@ BASE_URL = 'https://binaries.spack.io/' MANIFEST_URL = BASE_URL + 'cache_spack_io_index.json' DATA_DIR = Path(__file__).parent / '_data' -PACKAGE_DATA_PATH = DATA_DIR / 'package_data.json' -SPECS_DATA_PATH = DATA_DIR / 'specs_data.json' -TREE_DATA_PATH = DATA_DIR / 'tree_data.json' +PACKAGE_DATA_PATH = DATA_DIR / 'package_data.json.gz' +SPECS_DATA_PATH = DATA_DIR / 'specs_data.json.gz' +TREE_DATA_PATH = DATA_DIR / 'tree_data.json.gz' class SetEncoder(JSONEncoder): @@ -28,7 +29,7 @@ def get_response(url): def save_data(data, path): path.parent.mkdir(exist_ok=True, parents=True) - with open(path, 'w') as f: + with gzip.open(path, 'wt', encoding='UTF-8') as f: json.dump(data, f, indent=4, cls=SetEncoder) diff --git a/serve_static.py b/serve_static.py index fe609d8..381e66c 100644 --- a/serve_static.py +++ b/serve_static.py @@ -1,4 +1,4 @@ -from http.server import SimpleHTTPRequestHandler, HTTPServer +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer import click from pathlib import Path @@ -19,7 +19,7 @@ def __init__(self, *args, **kwargs): ) def run_server(port): server_address = ('', port) - with HTTPServer(server_address, CustomHandler) as httpd: + with ThreadingHTTPServer(server_address, CustomHandler) as httpd: print(f"Serving at http://localhost:{port}") httpd.serve_forever() diff --git a/templates/static/script.js b/templates/static/script.js index a66b1a9..c05485a 100644 --- a/templates/static/script.js +++ b/templates/static/script.js @@ -29,6 +29,14 @@ const noDiffMessage = '-'; // General +async function fetchGzippedJson(url) { + const response = await fetch(url); + const ds = new DecompressionStream('gzip'); + const decompressedStream = response.body.pipeThrough(ds); + const text = await new Response(decompressedStream).text(); + return JSON.parse(text); +} + function navigateToHome() { window.history.pushState(null, '', basePath + '/'); } @@ -46,11 +54,18 @@ function applyRoute(params) { // https://datatables.net/manual/tech-notes/3 setupDataTable(); } - badgeFilters = Object.fromEntries( - Object.keys(badgeFilters).map((key) => [key, urlParams.getAll(key)]) - ) - badgeFiltersUpdated(); - updateTable(); + if (specData) { + badgeFilters = Object.fromEntries( + Object.keys(badgeFilters).map((key) => [key, urlParams.getAll(key)]) + ) + badgeFiltersUpdated(); + updateTable(); + } else { + // Set table empty message + Array.from(document.getElementsByClassName('dt-empty')).forEach( + (el) => el.innerHTML = 'Loading data...' + ) + } } } showContent(contentToShow); @@ -75,8 +90,10 @@ function showContent(content_id) { } function setPackageName(name) { - currentSpecs = packageData[packageName].specs.map((hash) => specData[hash]); - updateBadgeOptions(); + if (specData) { + currentSpecs = packageData[packageName].specs.map((hash) => specData[hash]); + updateBadgeOptions(); + } document.getElementById('package-name').innerHTML = name; document.getElementById('package-link').href = "https://packages.spack.io/package.html?name=" + name; } @@ -647,17 +664,25 @@ function updateTable() { // Ready $(document).ready(async function () { basePath = document.getElementById('base-path').innerHTML; - packageData = await (await fetch(`${basePath}/api/package_data.json`)).json(); - specData = await (await fetch(`${basePath}/api/specs_data.json`)).json(); - treeData = await (await fetch(`${basePath}/api/tree_data.json`)).json(); - applyRoute(window.location.search); + fetchGzippedJson(`${basePath}/api/tree_data.json.gz`).then((data) => { + treeData = data + loadTree('Stack -> Tag'); + filterTree(); + }); + fetchGzippedJson(`${basePath}/api/package_data.json.gz`).then((data) => { + packageData = data + applyRoute(window.location.search); + }); + fetchGzippedJson(`${basePath}/api/specs_data.json.gz`).then((data) => { + specData = data + applyRoute(window.location.search); + }); + window.navigation.addEventListener("navigate", (e) => { const dest = e.destination.url; applyRoute(dest.includes('?') ? dest.split('?')[1] : '') }); - loadTree('Stack -> Tag'); - filterTree(); setupSidebarResize(); })