Preemptive loading

src/components/RidgelinePlot.vue

@@ -23,10 +23,30 @@
 import useData from '@/composables/useData';
 import { useAppStore } from '@/stores/appStore';
 import { Dimensions } from '@/types';
+import { watch } from 'vue';
 
 const appStore = useAppStore();
 
-const { fetchErrors, geographicalResolutions, histogramData } = useData();
+const { fetchErrors, geographicalResolutions, histogramData, preemptiveLoad } = useData();
+
+const largestDataFilesInSizeOrder = [
+  "hist_counts_deaths_disease_activity_type_country_log.json", // 19.6MB
+  "hist_counts_dalys_disease_activity_type_country_log.json", // 19.4MB
+  "hist_counts_dalys_disease_activity_type_country.json", // 18.3MB
+  "hist_counts_deaths_disease_activity_type_country.json", // 17.9MB
+  "hist_counts_deaths_disease_country_log.json", // 9.7MB
+  "hist_counts_dalys_disease_country_log.json", // 9.5MB
+  "hist_counts_dalys_disease_country.json", // 8.7MB
+  "hist_counts_deaths_disease_country.json", // 8.7MB
+  // Next largest file is 3.6MB, so we stop here.
+]
+
+watch(histogramData, () => {
+  // Once the first user-requested data has loaded (histogramData),
+  // start preemptive loading of the largest data files.
+  // Don't await: this should be a background process, not block anything.
+  preemptiveLoad(largestDataFilesInSizeOrder)
+});
 </script>
 
 <style lang="css" scoped>

src/composables/useData.ts

@@ -1,7 +1,7 @@
 import { useAppStore } from "@/stores/appStore";
 import { type DataRow, Dimensions, LocResolutions } from "@/types";
 import { debounce } from "perfect-debounce";
-import { computed, ref, shallowRef, watch } from "vue";
+import { computed, onUnmounted, ref, shallowRef, watch } from "vue";
 
 import countryOptions from '@/data/options/countryOptions.json';
 import subregionOptions from '@/data/options/subregionOptions.json';
@@ -14,6 +14,11 @@ export default () => {
   const histogramData = ref<DataRow[]>([]);
   const histogramDataCache = shallowRef<Record<string, DataRow[]>>({});
 
+  // Because some files are very large (up to 20MB), we support preemptive background loading of files.
+  // preemptiveControllers tracks the abort controllers, so that these preemptive fetches can be cancelled
+  // when we need to prioritize user-requested data loading.
+  const preemptiveControllers = new Map<string, AbortController>();
+
   // The geographical resolutions to use based on current exploreBy and focus selections.
   // This is currently exposed by the composable but that's only for manual testing purposes.
   const geographicalResolutions = computed(() => {
@@ -54,28 +59,61 @@ export default () => {
     });
   });
 
-  // Fetch and parse multiple JSONs, and merge together all data.
+  // Cancel all preemptive fetches (call this when user requests specific data)
+  const cancelPreemptiveFetches = () => {
+    console.error("I AM DOING A CANCEL for each of ", Array.from(preemptiveControllers.keys()));
+    preemptiveControllers.forEach((controller) => {
+      controller.abort();
+    });
+    preemptiveControllers.clear();
+  };
+
+  // Fetch a single file with optional abort support
+  const fetchWithAbort = async (path: string, signal?: AbortSignal): Promise<undefined> => {
+    if (histogramDataCache.value[path]) {
+      return;
+    }
+
+    try {
+      const response = await fetch(`${dataDir}/${path}`, { signal });
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      const rows = await response.json();
+      histogramDataCache.value = { ...histogramDataCache.value, [path]: rows };
+    } catch (error) {
+      if ((error as Error).name === 'AbortError') {
+        // Silently ignore cancelled requests
+        return;
+      }
+      fetchErrors.value.push({ e: error as Error, message: `Error loading data from path: ${path}. ${error}` });
+    }
+  };
+
+  // Fetch and parse multiple JSONs with priority (cancels preemptive fetches first)
   const loadDataFromPaths = async (paths: string[]) => {
     fetchErrors.value = [];
-    await Promise.all(paths.map(async (path) => {
-      if (!histogramDataCache.value[path]) {
-        try {
-          const response = await fetch(`${dataDir}/${path}`);
-          if (!response.ok) {
-            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-          }
-          const rows = await response.json();
-          histogramDataCache.value[path] = rows;
-          return rows;
-        } catch (error) {
-          fetchErrors.value.push({ e: error as Error, message: `Error loading data from path: ${path}. ${error}` });
-        }
-      }
-    }));
+    cancelPreemptiveFetches(); // Cancel background fetches to prioritize user request
 
+    await Promise.all(paths.map((path) => fetchWithAbort(path)));
     histogramData.value = paths.flatMap((path) => histogramDataCache.value[path] || []);
   };
 
+  // Preemptively load files in the background (cancelled when user requests data load)
+  const preemptiveLoad = async (paths: string[]) => {
+    for (const path of paths) {
+      if (histogramDataCache.value[path] || preemptiveControllers.has(path)) {
+        continue; // Already cached or already loading
+      }
+
+      const controller = new AbortController();
+      preemptiveControllers.set(path, controller);
+
+      await fetchWithAbort(path, controller.signal);
+      preemptiveControllers.delete(path);
+    }
+  };
+
   const doLoadData = debounce(async () => {
     await loadDataFromPaths(histogramDataPaths.value);
   }, 25)
@@ -89,5 +127,9 @@ export default () => {
     }
   }, { immediate: true });
 
-  return { histogramData, fetchErrors, geographicalResolutions };
+  onUnmounted(() => {
+    cancelPreemptiveFetches();
+  });
+
+  return { histogramData, fetchErrors, geographicalResolutions, preemptiveLoad };
 }

tests/unit/composables/useData.spec.ts

@@ -1,5 +1,5 @@
 import { createPinia, setActivePinia } from 'pinia';
-import { it, expect, describe, beforeEach, vi, Mock } from 'vitest';
+import { afterEach, it, expect, describe, beforeEach, vi, Mock } from 'vitest';
 
 import { server } from '../mocks/server';
 import histCountsDeathsDiseaseLog from "@/../public/data/json/hist_counts_deaths_disease_log.json";
@@ -28,6 +28,10 @@ describe('useData', () => {
     setActivePinia(createPinia());
   });
 
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
   it('should initialize with correct data, and request correct data as store selections change', async () => {
     const fetchSpy = vi.spyOn(global, 'fetch')
     const appStore = useAppStore();
@@ -154,4 +158,36 @@ describe('useData', () => {
 
     expect(histogramData.value).toEqual([]);
   });
+
+  it('should cancel preemptive fetches when any user-requested fetch is triggered', async () => {
+    // Make the preemptive fetch take a long time
+    server.use(
+      http.get("/data/json/hist_counts_dalys_disease_country_log.json", async () => {
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+        return HttpResponse.json(histCountsDalysDiseaseCountryLog);
+      }),
+    );
+
+    const fetchSpy = vi.spyOn(global, 'fetch')
+    const abortSpy = vi.spyOn(AbortController.prototype, 'abort');
+
+    const appStore = useAppStore();
+    const { histogramData, preemptiveLoad } = useData();
+    expect(histogramData.value).toEqual([]);
+
+    // Initial data
+    await vi.waitFor(() => {
+      expect(histogramData.value).toHaveLength(histCountsDeathsDiseaseLog.length);
+    });
+
+    // Start preemptive loading (don't await).
+    // This request is the long-lasting one which we will cause to be cancelled.
+    preemptiveLoad(["hist_counts_dalys_disease_country_log.json"]);
+    expect(fetchSpy).toHaveBeenCalledWith("/data/json/hist_counts_dalys_disease_country_log.json", { signal: expect.any(AbortSignal) });
+
+    // Trigger a user-requested data load, which should cancel the preemptive fetch.
+    appStore.focus = "Central and Southern Asia";
+
+    await vi.waitFor(() => expect(abortSpy).toHaveBeenCalled());
+  });
 });