Fix for CT API pagination change, keep Cookies

.dockerignore

@@ -2,4 +2,5 @@
 !*.js
 !ctldap.yml
 !package.json
-!yarn.lock
+!yarn.lock
+!.yarn*

.gitignore

@@ -2,3 +2,11 @@
 /.idea/
 *.iml
 .env
+
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions

.yarnrc.yml

@@ -0,0 +1,7 @@
+nodeLinker: node-modules
+
+plugins:
+  - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs
+    spec: "@yarnpkg/plugin-interactive-tools"
+
+yarnPath: .yarn/releases/yarn-3.6.1.cjs

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+### 3.0.2
+- Fixed error due to changed ChurchTools API pagination behavior
+- Keep session cookies, which gains about 100 ms speedup
+- Updated `yarn`, `bcrypt` and `got`
+
 ### 3.0.1
 - Fixed scope of `ldap.filters.SubstringFilter.prototype.matches` (no arrow function...)
 - Updated `ldapjs` and `ldap-escape`

Dockerfile

@@ -7,6 +7,8 @@ USER node
 
 COPY --chown=node:node package.json .
 COPY --chown=node:node yarn.lock .
+COPY --chown=node:node .yarnrc.yml .
+COPY --chown=node:node .yarn ./.yarn
 RUN yarn install
 
 COPY --chown=node:node *.js .

README.md

@@ -1,4 +1,4 @@
-# ctldap 3.0.1 - LDAP Wrapper for ChurchTools
+# ctldap 3.0.2 - LDAP Wrapper for ChurchTools
 
 This software acts as an LDAP server for ChurchTools 3
 

ctldap-site.js

@@ -8,6 +8,7 @@ import got from "got";
 import bcrypt from "bcrypt";
 import argon2 from "argon2";
 import { CtldapConfig } from "./ctldap-config.js"
+import { CookieJar } from "tough-cookie";
 
 export class CtldapSite {
 
@@ -31,12 +32,17 @@ export class CtldapSite {
         this.api = got.extend({
             headers: { "Authorization": `Login ${site.apiToken}` },
             prefixUrl: `${site.ctUri.replace(/\/$/g, '')}/api`,
+            // Let us keep cookies, which may improve CT API performance.
+            // "undefined" is fine as "store" parameter here, it results in memory storage.
+            cookieJar: new CookieJar(undefined),
             responseType: 'json',
             resolveBodyOnly: true,
             http2: true
         });
         this.adminDn = this.fnUserDn(site.ldapUser);
-        this.CACHE = {};
+        this.CACHE = {
+            pagination: {}
+        };
         this.loginErrorCount = 0;
         this.loginBlockedDate = null;
 

ctldap.js

@@ -11,6 +11,14 @@ import fs from "fs";
 import ldap from "ldapjs";
 import { CtldapConfig } from "./ctldap-config.js";
 
+/**
+ * Simple integer range as array, inspired by https://developer.mozilla.org
+ * @param start Start integer, inclusive
+ * @param end Stop integer, exclusive
+ * @return {number[]} Array with number sequence
+ */
+const range = (start, end) => Array.from({ length: end - start }, (_, i) => start + i);
+
 const parseDN = ldap.parseDN;
 const config = new CtldapConfig();
 
@@ -93,29 +101,41 @@ function getCached(site, key, factory) {
  * @param {string} apiPath The API endpoint to query for all paginated data.
  * @param {object} [searchParams] Additional search params (query parameters)
  */
-async function fetchAllPaginatedHack(site, apiPath, searchParams) {
-  // Get all records except the last one
-  const result = await site.api.get(apiPath, {
-    searchParams: {
-      ...(searchParams || {}),
-      limit: -1
-    }
-  });
-  const data = result['data'];
-  const total = result['meta']['pagination']['total'];
-  if (data.length < total) {
-    // Fetch last record and append it to the result
-    const limit = total - data.length;
-    const last = await site.api.get(apiPath, {
+async function fetchAllPaginated(site, apiPath, searchParams= {}) {
+  // Closure for fetching a single page
+  const fetchPage = (page) => {
+    return site.api.get(apiPath, {
       searchParams: {
-        ...(searchParams || {}),
-        limit,
-        page: Math.ceil(total / limit)
+        ...searchParams,
+        page
       }
     });
-    data.push(last['data'][0]);
+  };
+  // Get pagination meta cache
+  const pCache = site.CACHE.pagination;
+  // Assume the same number of pages as last time, default to 1
+  const assumedPages = pCache[site] || 1;
+  // Fetch assumed number of pages
+  const promises = range(1, assumedPages + 1).map(fetchPage);
+  // Await first result
+  const firstResult = await Promise.any(promises);
+  // Check first result for completeness, and fix up results and pagination cache if necessary
+  const nPages = firstResult['meta']['pagination']['lastPage'];
+  if (nPages !== assumedPages) {
+    logDebug(site, `Assumed ${assumedPages} page(s) of data for /api/${apiPath}, but had to load ${nPages}.`);
+    // Update meta cache
+    pCache[site] = nPages;
+    // Fetch remaining pages, if any
+    if (nPages > assumedPages) {
+      promises.push(...range(assumedPages + 1, nPages + 1).map(fetchPage));
+    }
+  } else {
+    logDebug(site, `Assumed ${assumedPages} page(s) of data for /api/${apiPath}, which was correct.`);
   }
-  return data;
+  // Await all results
+  const results = await Promise.all(promises);
+  // Collect all data via flatMap() and return it
+  return results.flatMap(r => r['data']);
 }
 
 /**
@@ -136,7 +156,7 @@ async function fetchMemberships(site) {
  * @param {object} site The site for which this information is requested.
  */
 async function fetchPersons(site) {
-  const data = await fetchAllPaginatedHack(site, 'persons');
+  const data = await fetchAllPaginated(site, 'persons', { limit: 500 });
   logDebug(site, "fetchPersons done");
   const personMap = {};
   data.forEach((p) => {
@@ -153,7 +173,7 @@ async function fetchPersons(site) {
  * @param {object} site The site for which this information is requested.
  */
 async function fetchGroups(site) {
-  const data = await fetchAllPaginatedHack(site, 'groups');
+  const data = await fetchAllPaginated(site, 'groups', { limit: 100 });
   logDebug(site, "fetchGroups done");
   const groupMap = {};
   const sgmKeys = Object.keys(site.specialGroupMappings);

package.json

@@ -7,14 +7,16 @@
   "type": "module",
   "dependencies": {
     "argon2": "^0.30.3",
-    "bcrypt": "^5.0.0",
-    "got": "^12.5.3",
+    "bcrypt": "^5.1.0",
+    "got": "^13.0.0",
     "ldap-escape": "^2.0.6",
     "ldap-filter": "^0.3.3",
     "ldapjs": "^2.3.3",
-    "yaml-env-defaults": "^2.0.2"
+    "tough-cookie": "^4.1.3",
+    "yaml-env-defaults": "^2.0.5"
   },
   "scripts": {
     "start": "node ctldap.js"
-  }
+  },
+  "packageManager": "[email protected]"
 }