WIP: Support generating sitemap for SEO

config.sample.yml

@@ -145,3 +145,9 @@ dataPath: ./data
 # file uploads.
 
 bodyParserLimit: 5mb
+
+
+seo:
+ sitemap:
+ enabled: false # sitemap is experimental feature, do not enable in production
+ cacheExpireTime: 3600 # in seconds

package.json

@@ -176,6 +176,7 @@
  "semver": "7.3.8",
  "serve-favicon": "2.5.0",
  "simple-git": "3.16.0",
+ "sitemap": "7.1.1",
  "solr-node": "1.2.1",
  "sqlite3": "5.1.4",
  "ssh2": "1.11.0",

server/controllers/common.js

@@ -5,6 +5,7 @@ const _ = require('lodash')
 const CleanCSS = require('clean-css')
 const moment = require('moment')
 const qs = require('querystring')
+const { SitemapStream, streamToPromise } = require('sitemap')
 
 /* global WIKI */
 
@@ -22,6 +23,91 @@ router.get('/robots.txt', (req, res, next) => {
  }
 })
 
+/**
+ * sitemap.xml
+ */
+// TODO: toggle off before the feature is ready
+if(WIKI.config.seo.sitemap.enabled) {
+ WIKI.logger.info(`Experimental feature sitemap is enabled`)
+ router.get('/sitemap.xml', async (req, res
+
+ ) => {
+ const host = WIKI.config.host
+ const {enabled, cacheExpireTime} = WIKI.config.seo.sitemap
+ !enabled && res.status(404).end()
+
+ try {
+
+ res.header('Content-Type', 'application/xml');
+
+ // TODO: if we have a cached entry send it
+ // if (sitemap) {
+ // res.send(sitemap)
+ // return
+ // }
+
+ const smStream = new SitemapStream({ hostname: host})
+
+ /**
+ * [
+ Page {
+ id: 1,
+ path: 'home',
+ hash: 'b29b5d2ce62e55412776ab98f05631e0aa96597b',
+ title: 'Untitled Page',
+ description: '',
+ isPrivate: 0,
+ isPublished: 1,
+ privateNS: null,
+ publishStartDate: '',
+ publishEndDate: '',
+ content: '# Header\nYour content here',
+ render: '<h1 class="toc-header" id="header"><a href="#header" class="toc-anchor">¶</a> Header</h1>\n' +
+ '<p>Your content here</p>\n',
+ toc: '[{"title":"Header","anchor":"#header","children":[]}]',
+ contentType: 'markdown',
+ createdAt: '2023-10-12T16:07:42.594Z',
+ updatedAt: '2023-10-12T16:07:44.791Z',
+ editorKey: 'markdown',
+ localeCode: 'en',
+ authorId: 1,
+ creatorId: 1,
+ extra: { js: '', css: '' }
+ }
+ ]
+ */
+ const pages = await WIKI.models.pages.query()
+ .where('isPublished', 1)
+ .where('isPrivate', 0)
+ .orderBy('updatedAt', 'desc')
+
+ for (const page of pages) {
+ // TODO: 
+ /**
+ * TypeError: user.getGlobalPermissions is not a function
+ at Object.checkAccess (/home/xxx/workspace/wiki/server/core/auth.js:218:72)
+ at /home/xxx/workspace/wiki/server/controllers/common.js:92:22
+ */
+ // if (!WIKI.auth.checkAccess(WIKI.auth.guest, ['read:pages'], page)) continue
+
+ smStream.write({ 
+ url: `/${page.localeCode}/${page.path}`, 
+ lastmod: page.updatedAt
+ })
+ }
+
+ smStream.end();
+ smStream.pipe(res).on('error', (e) => { throw e })
+ } catch (e) {
+ console.error(e)
+ res.status(500).end()
+ }
+
+ })
+}
+
+
+
 /**
  * Health Endpoint
  */

server/setup.js

@@ -117,7 +117,11 @@ module.exports = () => {
  description: '',
  robots: ['index', 'follow'],
  analyticsService: '',
- analyticsId: ''
+ analyticsId: '',
+ sitemap: {
+ enabled: false,
+ cacheExpireTime: 3600 // in seconds
+ }
  })
  _.set(WIKI.config, 'sessionSecret', (await crypto.randomBytesAsync(32)).toString('hex'))
  _.set(WIKI.config, 'telemetry', {

yarn.lock

@@ -3839,6 +3839,11 @@
  resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.14.tgz#1c1d6e3c75dba466e0326948d56e8bd72a1903d2"
  integrity sha512-u/SJDyXwuihpwjXy7hOOghagLEV1KdAST6syfnOk6QZAMzZuWZqXy5aYYZbh8Jdpd4escVFP0MvftHNDb9pruA==
 
+"@types/node@^17.0.5":
+ version "17.0.45"
+ resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190"
+ integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==
+
 "@types/node@^8.0.47":
  version "8.10.53"
  resolved "https://registry.yarnpkg.com/@types/node/-/node-8.10.53.tgz#5fa08eef810b08b2c03073e360b54f7bad899db1"
@@ -3890,6 +3895,13 @@
  "@types/node" "*"
  safe-buffer "*"
 
+"@types/sax@^1.2.1":
+ version "1.2.5"
+ resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.5.tgz#4392799e1770d24b6dc8d0c66c8882f8e1c38b3d"
+ integrity sha512-9jWta97bBVC027/MShr3gLab8gPhKy4l6qpb+UJLF5pDm3501NvA7uvqVCW+REFtx00oTi6Cq9JzLwgq6evVgw==
+ dependencies:
+ "@types/node" "*"
+
 "@types/serve-static@*":
  version "1.13.3"
  resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.3.tgz#eb7e1c41c4468272557e897e9171ded5e2ded9d1"
@@ -5107,6 +5119,11 @@ are-we-there-yet@~1.1.2:
  delegates "^1.0.0"
  readable-stream "^2.0.6"
 
+arg@^5.0.0:
+ version "5.0.2"
+ resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c"
+ integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==
+
 argparse@^1.0.7:
  version "1.0.10"
  resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
@@ -18189,6 +18206,16 @@ sisteransi@^1.0.3:
  resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.3.tgz#98168d62b79e3a5e758e27ae63c4a053d748f4eb"
  integrity sha512-SbEG75TzH8G7eVXFSN5f9EExILKfly7SUvVY5DhhYLvfhKqhDFY0OzevWa/zwak0RLRfWS5AvfMWpd9gJvr5Yg==
 
+[email protected]:
+ version "7.1.1"
+ resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-7.1.1.tgz#eeed9ad6d95499161a3eadc60f8c6dce4bea2bef"
+ integrity sha512-mK3aFtjz4VdJN0igpIJrinf3EO8U8mxOPsTBzSsy06UtjZQJ3YY3o3Xa7zSc5nMqcMrRwlChHZ18Kxg0caiPBg==
+ dependencies:
+ "@types/node" "^17.0.5"
+ "@types/sax" "^1.2.1"
+ arg "^5.0.0"
+ sax "^1.2.4"
+
 slash@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"