Allow addProtocol in the worker code (#3459)

* Allow add protocol in the worker code

* Simplify API - make it more genereic and "low level blocks"

* Update changelog

* Update docs

* Improve docs

* docs

* Move all operations to the same file

* Rename file to protocol_crud

* Add more time to tests

CHANGELOG.md

@@ -1,7 +1,7 @@
 ## main
 
 ### ✨ Features and improvements
-
+- ⚠️ Add the ability to import a script in the worker thread and call `addProtocol` and `removeProtocol` there ([#3459](https://github.com/maplibre/maplibre-gl-js/pull/3459)) - this also changed how `addSrouceType` works since now you'll need to load the script with `maplibregl.importScriptInWorkers`.
 - Upgraded to use Node JS 20 and removed the dependency of `gl` package from the tests to allow easier develpment setup.
 - Improved precision and added a subtle fade transition to marker opacity changes ([#3431](https://github.com/maplibre/maplibre-gl-js/pull/3431))
 - _...Add new stuff here..._

src/index.ts

@@ -15,7 +15,7 @@ import {LngLatBounds} from './geo/lng_lat_bounds';
 import Point from '@mapbox/point-geometry';
 import {MercatorCoordinate} from './geo/mercator_coordinate';
 import {Evented} from './util/evented';
-import {AddProtocolAction, config} from './util/config';
+import {config} from './util/config';
 import {Debug} from './util/debug';
 import {isSafari} from './util/util';
 import {rtlMainThreadPluginFactory} from './source/rtl_text_plugin_main_thread';
@@ -31,6 +31,8 @@ import {RasterTileSource} from './source/raster_tile_source';
 import {VectorTileSource} from './source/vector_tile_source';
 import {VideoSource} from './source/video_source';
 import {addSourceType, type SourceClass} from './source/source';
+import {addProtocol, removeProtocol} from './source/protocol_crud';
+import {getGlobalDispatcher} from './util/dispatcher';
 const version = packageJSON.version;
 
 export type * from '@maplibre/maplibre-gl-style-spec';
@@ -179,47 +181,44 @@ class MapLibreGL {
     }
 
     /**
-     * Sets a custom load tile function that will be called when using a source that starts with a custom url schema.
+     * Adds a custom load resource function that will be called when using a URL that starts with a custom url schema.
+     * This will happen in the main thread, and workers might call it if they don't know how to handle the protocol.
      * The example below will be triggered for custom:// urls defined in the sources list in the style definitions.
-     * The function passed will receive the request parameters and should call the callback with the resulting request,
+     * The function passed will receive the request parameters and should return with the resulting resource,
      * for example a pbf vector tile, non-compressed, represented as ArrayBuffer.
      *
      * @param customProtocol - the protocol to hook, for example 'custom'
      * @param loadFn - the function to use when trying to fetch a tile specified by the customProtocol
      * @example
-     * This will fetch a file using the fetch API (this is obviously a non interesting example...)
      * ```ts
+     * // This will fetch a file using the fetch API (this is obviously a non interesting example...)
      * maplibregl.addProtocol('custom', async (params, abortController) => {
-            const t = await fetch(`https://${params.url.split("://")[1]}`);
-            if (t.status == 200) {
-                const buffer = await t.arrayBuffer();
-                return {data: buffer}
-            } else {
-                throw new Error(`Tile fetch error: ${t.statusText}`));
-            }
-        });
+     *      const t = await fetch(`https://${params.url.split("://")[1]}`);
+     *      if (t.status == 200) {
+     *          const buffer = await t.arrayBuffer();
+     *          return {data: buffer}
+     *      } else {
+     *          throw new Error(`Tile fetch error: ${t.statusText}`);
+     *      }
+     *  });
      * // the following is an example of a way to return an error when trying to load a tile
      * maplibregl.addProtocol('custom2', async (params, abortController) => {
      *      throw new Error('someErrorMessage'));
      * });
      * ```
      */
-    static addProtocol(customProtocol: string, loadFn: AddProtocolAction) {
-        config.REGISTERED_PROTOCOLS[customProtocol] = loadFn;
-    }
+    static addProtocol = addProtocol;
 
     /**
-     * Removes a previously added protocol
+     * Removes a previously added protocol in the main thread.
      *
      * @param customProtocol - the custom protocol to remove registration for
      * @example
      * ```ts
      * maplibregl.removeProtocol('custom');
      * ```
      */
-    static removeProtocol(customProtocol: string) {
-        delete config.REGISTERED_PROTOCOLS[customProtocol];
-    }
+    static removeProtocol = removeProtocol;
 
     /**
      * Adds a [custom source type](#Custom Sources), making it available for use with
@@ -229,6 +228,41 @@ class MapLibreGL {
      * @returns a promise that is resolved when the source type is ready or with an error argument if there is an error.
      */
     static addSourceType = (name: string, sourceType: SourceClass) => addSourceType(name, sourceType);
+
+    /**
+     * Allows loading javascript code in the worker thread.
+     * *Note* that since this is using some very internal classes and flows it is considered experimental and can break at any point.
+     *
+     * It can be useful for the following examples:
+     * 1. Using `self.addProtocol` in the worker thread - note that you might need to also register the protocol on the main thread.
+     * 2. Using `self.registerWorkerSource(workerSource: WorkerSource)` to register a worker source, which sould come with `addSourceType` usually.
+     * 3. using `self.actor.registerMessageHandler` to override some internal worker operations
+     * @param workerUrl - the worker url e.g. a url of a javascript file to load in the worker
+     * @returns
+     *
+     * @example
+     * ```ts
+     * // below is an example of sending a js file to the worker to load the method there
+     * // Note that you'll need to call the global function `addProtocol` in the worker to register the protocol there.
+     * // add-protocol-worker.js
+     * async function loadFn(params, abortController) {
+     *     const t = await fetch(`https://${params.url.split("://")[1]}`);
+     *     if (t.status == 200) {
+     *         const buffer = await t.arrayBuffer();
+     *         return {data: buffer}
+     *     } else {
+     *         throw new Error(`Tile fetch error: ${t.statusText}`);
+     *     }
+     * }
+     * self.addPRotocol('custom', loadFn);
+     *
+     * // main.js
+     * maplibregl.importScriptInWorkers('add-protocol-worker.js');
+     * ```
+     */
+    static importScriptInWorkers = (workerUrl: string) => {
+        return getGlobalDispatcher().broadcast('importScript', workerUrl);
+    };
 }
 
 //This gets automatically stripped out in production builds.

src/source/protocol_crud.ts

@@ -0,0 +1,13 @@
+import {AddProtocolAction, config} from '../util/config';
+
+export function getProtocol(url: string) {
+    return config.REGISTERED_PROTOCOLS[url.substring(0, url.indexOf('://'))];
+}
+
+export function addProtocol(customProtocol: string, loadFn: AddProtocolAction) {
+    config.REGISTERED_PROTOCOLS[customProtocol] = loadFn;
+}
+
+export function removeProtocol(customProtocol: string) {
+    delete config.REGISTERED_PROTOCOLS[customProtocol];
+}

src/source/source.test.ts

@@ -28,16 +28,6 @@ describe('addSourceType', () => {
         expect(sourceType).toHaveBeenCalled();
     });
 
-    test('triggers workers to load worker source code', async () => {
-        const sourceType = function () {} as any as SourceClass;
-        sourceType.workerSourceURL = 'worker-source.js' as any as URL;
-
-        const spy = jest.spyOn(Dispatcher.prototype, 'broadcast');
-
-        await addSourceType('bar', sourceType);
-        expect(spy).toHaveBeenCalledWith('loadWorkerSource', 'worker-source.js');
-    });
-
     test('refuses to add new type over existing name', async () => {
         const sourceType = function () {} as any as SourceClass;
         await expect(addSourceType('canvas', sourceType)).rejects.toThrow();

src/source/source.ts

@@ -5,7 +5,7 @@ import {GeoJSONSource} from '../source/geojson_source';
 import {VideoSource} from '../source/video_source';
 import {ImageSource} from '../source/image_source';
 import {CanvasSource} from '../source/canvas_source';
-import {Dispatcher, getGlobalDispatcher} from '../util/dispatcher';
+import {Dispatcher} from '../util/dispatcher';
 
 import type {SourceSpecification} from '@maplibre/maplibre-gl-style-spec';
 import type {Event, Evented} from '../util/evented';
@@ -118,23 +118,12 @@ export interface Source {
     prepare?(): void;
 }
 
-/**
- * A supporting type to the source definition
- */
-type SourceStatics = {
-    /*
-     * An optional URL to a script which, when run by a Worker, registers a {@link WorkerSource}
-     * implementation for this Source type by calling `self.registerWorkerSource(workerSource: WorkerSource)`.
-     */
-    workerSourceURL?: URL;
-};
-
 /**
  * A general definition of a {@link Source} class for factory usage
  */
 export type SourceClass = {
     new (id: string, specification: SourceSpecification | CanvasSourceSpecification, dispatcher: Dispatcher, eventedParent: Evented): Source;
-} & SourceStatics;
+}
 
 /**
  * Creates a tiled data source instance given an options object.
@@ -187,12 +176,5 @@ export const addSourceType = async (name: string, SourceType: SourceClass): Prom
     if (getSourceType(name)) {
         throw new Error(`A source type called "${name}" already exists.`);
     }
-
     setSourceType(name, SourceType);
-
-    if (!SourceType.workerSourceURL) {
-        return;
-    }
-    const dispatcher = getGlobalDispatcher();
-    await dispatcher.broadcast('loadWorkerSource', SourceType.workerSourceURL.toString());
 };

src/source/worker.test.ts

@@ -125,7 +125,7 @@ describe('Worker register RTLTextPlugin', () => {
 
     test('calls callback on error', done => {
         const server = fakeServer.create();
-        worker.actor.messageHandlers['loadWorkerSource']('0', '/error').catch((err) => {
+        worker.actor.messageHandlers['importScript']('0', '/error').catch((err) => {
             expect(err).toBeTruthy();
             server.restore();
             done();

src/source/worker.ts

@@ -5,6 +5,7 @@ import {RasterDEMTileWorkerSource} from './raster_dem_tile_worker_source';
 import {rtlWorkerPlugin, RTLTextPlugin} from './rtl_text_plugin_worker';
 import {GeoJSONWorkerSource, LoadGeoJSONParameters} from './geojson_worker_source';
 import {isWorker} from '../util/util';
+import {addProtocol, removeProtocol} from './protocol_crud';
 
 import type {
     WorkerSource,
@@ -72,6 +73,9 @@ export default class Worker {
             this.externalWorkerSourceTypes[name] = WorkerSource;
         };
 
+        this.self.addProtocol = addProtocol;
+        this.self.removeProtocol = removeProtocol;
+
         // This is invoked by the RTL text plugin when the download via the `importScripts` call has finished, and the code has been parsed.
         this.self.registerRTLTextPlugin = (rtlTextPlugin: RTLTextPlugin) => {
             if (rtlWorkerPlugin.isParsed()) {
@@ -143,7 +147,7 @@ export default class Worker {
             return this._syncRTLPluginState(mapId, params);
         });
 
-        this.actor.registerMessageHandler('loadWorkerSource', async (_mapId: string, params: string) => {
+        this.actor.registerMessageHandler('importScript', async (_mapId: string, params: string) => {
             this.self.importScripts(params);
         });
 

src/util/actor_messages.ts

@@ -100,7 +100,7 @@ export type RequestResponseMessageMap = {
     'syncRTLPluginState': [PluginState, boolean];
     'setReferrer': [string, void];
     'removeSource': [RemoveSourceParams, void];
-    'loadWorkerSource': [string, void];
+    'importScript': [string, void];
     'removeTile': [TileParameters, void];
     'abortTile': [TileParameters, void];
     'removeDEMTile': [TileParameters, void];

src/util/ajax.ts

@@ -1,6 +1,6 @@
 import {extend, isWorker} from './util';
-import {config} from './config';
 import {createAbortError} from './abort_error';
+import {getProtocol} from '../source/protocol_crud';
 
 /**
  * This is used to identify the global dispatcher id when sending a message from the worker without a target map id.
@@ -129,8 +129,6 @@ export const getReferrer = () => isWorker(self) ?
     self.worker && self.worker.referrer :
     (window.location.protocol === 'blob:' ? window.parent : window).location.href;
 
-export const getProtocolAction = (url: string) => config.REGISTERED_PROTOCOLS[url.substring(0, url.indexOf('://'))];
-
 /**
  * Determines whether a URL is a file:// URL. This is obviously the case if it begins
  * with file://. Relative URLs are also file:// URLs iff the original document was loaded
@@ -228,12 +226,13 @@ function makeXMLHttpRequest(requestParameters: RequestParameters, abortControlle
  */
 export const makeRequest = function(requestParameters: RequestParameters, abortController: AbortController): Promise<GetResourceResponse<any>> {
     if (/:\/\//.test(requestParameters.url) && !(/^https?:|^file:/.test(requestParameters.url))) {
+        const protocolLoadFn = getProtocol(requestParameters.url);
+        if (protocolLoadFn) {
+            return protocolLoadFn(requestParameters, abortController);
+        }
         if (isWorker(self) && self.worker && self.worker.actor) {
             return self.worker.actor.sendAsync({type: 'getResource', data: requestParameters, targetMapId: GLOBAL_DISPATCHER_ID}, abortController);
         }
-        if (!isWorker(self) && getProtocolAction(requestParameters.url)) {
-            return getProtocolAction(requestParameters.url)(requestParameters, abortController);
-        }
     }
     if (!isFileURL(requestParameters.url)) {
         if (fetch && Request && AbortController && Object.prototype.hasOwnProperty.call(Request.prototype, 'signal')) {

src/util/image_request.ts

@@ -1,9 +1,9 @@
-import {RequestParameters, makeRequest, sameOrigin, getProtocolAction, GetResourceResponse} from './ajax';
-
+import {RequestParameters, makeRequest, sameOrigin, GetResourceResponse} from './ajax';
 import {arrayBufferToImageBitmap, arrayBufferToImage, extend, isWorker, isImageBitmap} from './util';
 import {webpSupported} from './webp_supported';
 import {config} from './config';
 import {createAbortError} from './abort_error';
+import {getProtocol} from '../source/protocol_crud';
 
 type ImageQueueThrottleControlCallback = () => boolean;
 
@@ -154,7 +154,7 @@ export namespace ImageRequest {
         // - HtmlImageElement request automatically adds accept header for all the browser supported images
         const canUseHTMLImageElement = supportImageRefresh === false &&
             !isWorker(self) &&
-            !getProtocolAction(requestParameters.url) &&
+            !getProtocol(requestParameters.url) &&
             (!requestParameters.headers ||
                 Object.keys(requestParameters.headers).reduce((acc, item) => acc && item === 'accept', true));
 

src/util/web_worker.ts

@@ -1,11 +1,13 @@
-import {config} from './config';
+import {AddProtocolAction, config} from './config';
 import type {default as MaplibreWorker} from '../source/worker';
 import type {WorkerSourceConstructor} from '../source/worker_source';
 
 export interface WorkerGlobalScopeInterface {
     importScripts(...urls: Array<string>): void;
     registerWorkerSource: (sourceName: string, sourceConstrucor: WorkerSourceConstructor) => void;
     registerRTLTextPlugin: (_: any) => void;
+    addProtocol: (customProtocol: string, loadFn: AddProtocolAction) => void;
+    removeProtocol: (customProtocol: string) => void;
     worker: MaplibreWorker;
 }
 

test/integration/render/tests/text-variable-anchor-offset/pitched-offset/style.json

@@ -3,7 +3,10 @@
   "metadata": {
     "test": {
       "height": 256,
-      "operations": [["idle"]]
+      "operations": [
+        ["idle"],
+        ["wait", 500]
+      ]
     }
   },
   "center": [

test/integration/render/tests/text-variable-anchor-offset/pitched-with-map/style.json

@@ -5,7 +5,7 @@
       "height": 256,
       "operations": [
         ["idle"],
-        ["wait", 100]
+        ["wait", 500]
       ]
     }
   },

test/unit/lib/web_worker_mock.ts

@@ -1,13 +1,16 @@
+import MapLibreWorker from '../../../src/source/worker';
 import type {WorkerGlobalScopeInterface} from '../../../src/util/web_worker';
 import type {ActorTarget} from '../../../src/util/actor';
-import MapLibreWorker from '../../../src/source/worker';
 
 export class MessageBus implements WorkerGlobalScopeInterface, ActorTarget {
     addListeners: Array<EventListener>;
     postListeners: Array<EventListener>;
     target: MessageBus;
+
     registerWorkerSource: any;
     registerRTLTextPlugin: any;
+    addProtocol: any;
+    removeProtocol: any;
     worker: any;
 
     constructor(addListeners: Array<EventListener>, postListeners: Array<EventListener>) {