Async data loading

data.py

@@ -2,6 +2,7 @@
 import requests
 import time
 import json
+import gzip
 from json import JSONEncoder
 from pathlib import Path
 from functools import lru_cache
@@ -10,9 +11,9 @@
 BASE_URL = 'https://binaries.spack.io/'
 MANIFEST_URL = BASE_URL + 'cache_spack_io_index.json'
 DATA_DIR = Path(__file__).parent / '_data'
-PACKAGE_DATA_PATH = DATA_DIR / 'package_data.json'
-SPECS_DATA_PATH = DATA_DIR / 'specs_data.json'
-TREE_DATA_PATH = DATA_DIR / 'tree_data.json'
+PACKAGE_DATA_PATH = DATA_DIR / 'package_data.json.gz'
+SPECS_DATA_PATH = DATA_DIR / 'specs_data.json.gz'
+TREE_DATA_PATH = DATA_DIR / 'tree_data.json.gz'
 
 
 class SetEncoder(JSONEncoder):
@@ -28,7 +29,7 @@ def get_response(url):
 
 def save_data(data, path):
     path.parent.mkdir(exist_ok=True, parents=True)
-    with open(path, 'w') as f:
+    with gzip.open(path, 'wt', encoding='UTF-8') as f:
         json.dump(data, f, indent=4, cls=SetEncoder)
 
 

templates/static/script.js

@@ -29,6 +29,14 @@ const noDiffMessage = '-';
 
 
 // General
+async function fetchGzippedJson(url) {
+  const response = await fetch(url);
+  const ds = new DecompressionStream('gzip');
+  const decompressedStream = response.body.pipeThrough(ds);
+  const text = await new Response(decompressedStream).text();
+  return JSON.parse(text);
+}
+
 function navigateToHome() {
     window.history.pushState(null, '', basePath + '/');
 }
@@ -46,11 +54,18 @@ function applyRoute(params) {
                 // https://datatables.net/manual/tech-notes/3
                 setupDataTable();
             }
-            badgeFilters = Object.fromEntries(
-                Object.keys(badgeFilters).map((key) => [key, urlParams.getAll(key)])
-            )
-            badgeFiltersUpdated();
-            updateTable();
+            if (specData) {
+                badgeFilters = Object.fromEntries(
+                    Object.keys(badgeFilters).map((key) => [key, urlParams.getAll(key)])
+                )
+                badgeFiltersUpdated();
+                updateTable();
+            } else {
+                // Set table empty message
+                Array.from(document.getElementsByClassName('dt-empty')).forEach(
+                    (el) => el.innerHTML = 'Loading data...'
+                )
+            }
         }
     }
     showContent(contentToShow);
@@ -75,8 +90,10 @@ function showContent(content_id) {
 }
 
 function setPackageName(name) {
-    currentSpecs = packageData[packageName].specs.map((hash) => specData[hash]);
-    updateBadgeOptions();
+    if (specData) {
+        currentSpecs = packageData[packageName].specs.map((hash) => specData[hash]);
+        updateBadgeOptions();
+    }
     document.getElementById('package-name').innerHTML = name;
     document.getElementById('package-link').href = "https://packages.spack.io/package.html?name=" + name;
 }
@@ -647,17 +664,25 @@ function updateTable() {
 // Ready
 $(document).ready(async function () {
     basePath = document.getElementById('base-path').innerHTML;
-    packageData = await (await fetch(`${basePath}/api/package_data.json`)).json();
-    specData = await (await fetch(`${basePath}/api/specs_data.json`)).json();
-    treeData = await (await fetch(`${basePath}/api/tree_data.json`)).json();
 
-    applyRoute(window.location.search);
+    fetchGzippedJson(`${basePath}/api/tree_data.json.gz`).then((data) => {
+        treeData = data
+        loadTree('Stack -> Tag');
+        filterTree();
+    });
+    fetchGzippedJson(`${basePath}/api/package_data.json.gz`).then((data) => {
+        packageData = data
+        applyRoute(window.location.search);
+    });
+    fetchGzippedJson(`${basePath}/api/specs_data.json.gz`).then((data) => {
+        specData = data
+        applyRoute(window.location.search);
+    });
+
     window.navigation.addEventListener("navigate", (e) => {
         const dest = e.destination.url;
         applyRoute(dest.includes('?') ? dest.split('?')[1] : '')
     });
 
-    loadTree('Stack -> Tag');
-    filterTree();
     setupSidebarResize();
 })