hotosm · spwoodcock · Mar 27, 2024 · Mar 13, 2024 · Mar 13, 2024 · Mar 13, 2024
diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile
@@ -17,6 +17,7 @@
 ARG PYTHON_IMG_TAG=3.10
 ARG MINIO_TAG=${MINIO_TAG:-RELEASE.2024-01-01T16-36-33Z}
 FROM docker.io/minio/minio:${MINIO_TAG} as minio
+FROM docker.io/protomaps/go-pmtiles:v1.19.0 as go-pmtiles
 
 
 # Includes all labels and timezone info to extend from
@@ -107,6 +108,9 @@ RUN set -ex \
     && rm -rf /var/lib/apt/lists/*
 # Copy minio mc client
 COPY --from=minio /usr/bin/mc /usr/local/bin/
+# Copy go-pmtiles until for mbtiles-->pmtiles conversion
+# FIXME osm-fieldwork should do this, but is currently broken
+COPY --from=go-pmtiles /go-pmtiles /usr/local/bin/pmtiles
 COPY *-entrypoint.sh /
 ENTRYPOINT ["/app-entrypoint.sh"]
 # Copy Python deps from build to runtime

diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py
@@ -19,6 +19,7 @@
 
 import json
 import os
+import subprocess
 import uuid
 from asyncio import gather
 from concurrent.futures import ThreadPoolExecutor, wait
@@ -1769,8 +1770,7 @@ def get_project_tiles(
         tms (str, optional): Default None. Custom TMS provider URL.
     """
     zooms = "12-19"
-    tiles_path_id = uuid.uuid4()
-    tiles_dir = f"{TILESDIR}/{tiles_path_id}"
+    tiles_dir = f"{TILESDIR}/{project_id}"
     outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}"
 
     tile_path_instance = db_models.DbTilesPath(
@@ -1815,15 +1815,34 @@ def get_project_tiles(
             f"xy={False} | "
             f"tms={tms}"
         )
-        create_basemap_file(
-            boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}",
-            outfile=outfile,
-            zooms=zooms,
-            outdir=tiles_dir,
-            source=source,
-            xy=False,
-            tms=tms,
-        )
+
+        # TODO replace this temp workaround with osm-fieldwork code
+        # TODO to generate pmtiles directly instead of with go-pmtiles
+        if output_format == "pmtiles":
+            create_basemap_file(
+                boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}",
+                outfile=outfile.replace("pmtiles", "mbtiles"),
+                zooms=zooms,
+                outdir=tiles_dir,
+                source=source,
+                xy=False,
+                tms=tms,
+            )
+            subprocess.call(
+                "pmtiles convert " f"{outfile.replace('pmtiles', 'mbtiles')} {outfile}",
+                shell=True,
+            )
+        else:
+            create_basemap_file(
+                boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}",
+                outfile=outfile,
+                zooms=zooms,
+                outdir=tiles_dir,
+                source=source,
+                xy=False,
+                tms=tms,
+            )
+
         log.info(f"Basemap created for project ID {project_id}: {outfile}")
 
         tile_path_instance.status = 4

diff --git a/src/frontend/index.html b/src/frontend/index.html
@@ -11,6 +11,6 @@
 
   <body>
     <div id="app"></div>
-    <script type="module" src="/src/App.jsx"></script>
+    <script type="module" src="/src/App.tsx"></script>
   </body>
 </html>
diff --git a/src/frontend/package.json b/src/frontend/package.json
@@ -96,6 +96,7 @@
     "mini-css-extract-plugin": "^2.7.5",
     "ol-ext": "^4.0.11",
     "pako": "^2.1.0",
+    "pmtiles": "^3.0.5",
     "qrcode-generator": "^1.4.4",
     "react": "^17.0.2",
     "react-datepicker": "^6.1.0",

diff --git a/src/frontend/pnpm-lock.yaml b/src/frontend/pnpm-lock.yaml
diff --git a/src/frontend/src/App.jsx → src/frontend/src/App.tsx b/src/frontend/src/App.jsx → src/frontend/src/App.tsx
@@ -2,8 +2,9 @@ import axios from 'axios';
 import React, { useEffect } from 'react';
 import ReactDOM from 'react-dom';
 import { RouterProvider } from 'react-router-dom';
-import { Provider } from 'react-redux';
+import { Provider, useDispatch } from 'react-redux';
 import { PersistGate } from 'redux-persist/integration/react';
+import { CommonActions } from '@/store/slices/CommonSlice';
 
 import { store, persistor } from '@/store/Store';
 import routes from '@/routes';
@@ -13,6 +14,11 @@ import '@/index.css';
 import 'ol/ol.css';
 import 'react-loading-skeleton/dist/skeleton.css';
 
+enum Status {
+  'online',
+  'offline',
+}
+
 // Added Fix of Console Error of MUI Issue
 const consoleError = console.error;
 const SUPPRESSED_WARNINGS = [
@@ -51,9 +57,27 @@ axios.interceptors.request.use(
 );
 
 const GlobalInit = () => {
+  const dispatch = useDispatch();
+  const checkStatus = (status: string) => {
+    console.log(status);
+    dispatch(
+      CommonActions.SetSnackBar({
+        open: true,
+        message: 'Connection Status: ' + status,
+        variant: status === 'online' ? 'success' : 'error',
+        duration: 2000,
+      }),
+    );
+  };
   useEffect(() => {
+    window.addEventListener('offline', () => checkStatus('offline'));
+    window.addEventListener('online', () => checkStatus('online'));
+
     // Do stuff at init here
-    return () => {};
+    return () => {
+      window.removeEventListener('offline', () => checkStatus('offline'));
+      window.removeEventListener('online', () => checkStatus('online'));
+    };
   }, []);
   return null; // Renders nothing
 };

diff --git a/src/frontend/src/api/Files.js b/src/frontend/src/api/Files.js
@@ -54,3 +54,69 @@ export const ProjectFilesById = (odkToken, projectName, osmUser, taskId) => {
   }, [taskId]);
   return { qrcode };
 };
+
+export async function readFileFromOPFS(filePath) {
+  const opfsRoot = await navigator.storage.getDirectory();
+  const directories = filePath.split('/');
+
+  let currentDirectoryHandle = opfsRoot;
+
+  // Iterate dirs and get directoryHandles
+  for (const directory of directories.slice(0, -1)) {
+    console.log(`Reading OPFS dir: ${directory}`);
+    try {
+      currentDirectoryHandle = await currentDirectoryHandle.getDirectoryHandle(directory);
+    } catch {
+      return null; // Directory doesn't exist
+    }
+  }
+
+  // Get file within final directory handle
+  try {
+    const filename = directories.pop();
+    console.log(`Getting OPFS file: ${filename}`);
+    const fileHandle = await currentDirectoryHandle.getFileHandle(filename);
+    const fileData = await fileHandle.getFile(); // Read the file
+    return fileData;
+  } catch {
+    return null; // File doesn't exist or error occurred
+  }
+}
+
+export async function writeBinaryToOPFS(filePath, data) {
+  console.log(`Starting write to OPFS file: ${filePath}`);
+
+  const opfsRoot = await navigator.storage.getDirectory();
+
+  // Split the filePath into directories and filename
+  const directories = filePath.split('/');
+  const filename = directories.pop();
+
+  // Start with the root directory handle
+  let currentDirectoryHandle = opfsRoot;
+
+  // Iterate over directories and create nested directories
+  for (const directory of directories) {
+    console.log(`Creating OPFS dir: ${directory}`);
+    try {
+      currentDirectoryHandle = await currentDirectoryHandle.getDirectoryHandle(directory, { create: true });
+    } catch (error) {
+      console.error('Error creating directory:', error);
+    }
+  }
+
+  // Create the file handle within the last directory
+  const fileHandle = await currentDirectoryHandle.getFileHandle(filename, { create: true });
+  const writable = await fileHandle.createWritable();
+
+  // Write data to the writable stream
+  try {
+    await writable.write(data);
+  } catch (error) {
+    console.log(error);
+  }
+
+  // Close the writable stream
+  await writable.close();
+  console.log(`Finished write to OPFS file: ${filePath}`);
+}
diff --git a/src/frontend/src/api/Project.js b/src/frontend/src/api/Project.js
@@ -3,6 +3,7 @@ import { CommonActions } from '@/store/slices/CommonSlice';
 import CoreModules from '@/shared/CoreModules';
 import { task_priority_str } from '@/types/enums';
 import axios from 'axios';
+import { writeBinaryToOPFS } from '@/api/Files';
 
 export const ProjectById = (existingProjectList, projectId) => {
   return async (dispatch) => {
@@ -155,31 +156,50 @@ export const GenerateProjectTiles = (url, payload) => {
   };
 };
 
-export const DownloadTile = (url, payload) => {
+export const DownloadTile = (url, payload, toOpfs = false) => {
   return async (dispatch) => {
     dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: true }));
 
-    const getDownloadTile = async (url, payload) => {
+    const getDownloadTile = async (url, payload, toOpfs) => {
       try {
         const response = await CoreModules.axios.get(url, {
-          responseType: 'blob',
+          responseType: 'arraybuffer',
         });
 
         // Get filename from content-disposition header
+        const tileData = response.data;
+
+        if (toOpfs) {
+          // Copy to OPFS filesystem for offline use
+          const projectId = payload.id;
+          const filePath = `${projectId}/all.pmtiles`;
+          await writeBinaryToOPFS(filePath, tileData);
+          // Set the OPFS file path to project state
+          dispatch(ProjectActions.SetProjectOpfsBasemapPath(filePath));
+          return;
+        }
+
         const filename = response.headers['content-disposition'].split('filename=')[1];
+        // Create Blob from ArrayBuffer
+        const blob = new Blob([tileData], { type: response.headers['content-type'] });
+        const downloadUrl = URL.createObjectURL(blob);
 
-        var a = document.createElement('a');
-        a.href = window.URL.createObjectURL(response.data);
+        const a = document.createElement('a');
+        a.href = downloadUrl;
         a.download = filename;
         a.click();
+
+        // Clean up object URL
+        URL.revokeObjectURL(downloadUrl);
+
         dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: false }));
       } catch (error) {
         dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: false }));
       } finally {
         dispatch(ProjectActions.SetDownloadTileLoading({ type: payload, loading: false }));
       }
     };
-    await getDownloadTile(url, payload);
+    await getDownloadTile(url, payload, toOpfs);
   };
 };
 
@@ -206,7 +226,7 @@ export const GetProjectComments = (url) => {
     const getProjectComments = async (url) => {
       try {
         dispatch(ProjectActions.SetProjectGetCommentsLoading(true));
-        const response = await axios.get(url);
+        const response = await CoreModules.axios.get(url);
         dispatch(ProjectActions.SetProjectCommentsList(response.data));
         dispatch(ProjectActions.SetProjectGetCommentsLoading(false));
       } catch (error) {
@@ -224,7 +244,7 @@ export const PostProjectComments = (url, payload) => {
     const postProjectComments = async (url) => {
       try {
         dispatch(ProjectActions.SetPostProjectCommentsLoading(true));
-        const response = await axios.post(url, payload);
+        const response = await CoreModules.axios.post(url, payload);
         dispatch(ProjectActions.UpdateProjectCommentsList(response.data));
         dispatch(ProjectActions.SetPostProjectCommentsLoading(false));
       } catch (error) {