AG-12053 - Add archive folders to robots.txt and add noindex

ag-grid · Jul 3, 2024 · ee415a3 · ee415a3
1 parent dfd9289
commit ee415a3
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 4 deletions.
diff --git a/packages/ag-charts-website/src/layouts/Layout.astro b/packages/ag-charts-website/src/layouts/Layout.astro
@@ -7,7 +7,7 @@ import { Footer } from '@ag-website-shared/components/footer/Footer';
 import type { ApiMenuItem } from '@ag-grid-types';
 import { urlWithBaseUrl } from '@utils/urlWithBaseUrl';
 import { ApiTopBar } from '@components/top-bar/ApiTopBar';
-import { getIsProduction, getIsStaging } from '@utils/env';
+import { getIsProduction, getIsArchive, getIsStaging } from '@utils/env';
 import { isDynamicFrameworkPath, isDynamicFrameworkPathMatch, replaceDynamicFrameworkPath } from '@utils/framework';
 import Plausible from '../components/Plausible.astro';
 import { FRAMEWORKS } from '@constants';
@@ -37,6 +37,7 @@ const {
 const path = Astro.url.pathname;
 
 const isProduction = getIsProduction();
+const isArchive = getIsArchive();
 const isStaging = getIsStaging();
 const menuData = await getEntry('menu', 'data');
 const apiMenuItems = menuData.data.api.items as ApiMenuItem[];
@@ -95,7 +96,7 @@ const darkModeCSSUrl = `url("${urlWithBaseUrl('/images/moon.svg')}")`;
  <link rel="apple-touch-icon" sizes={`${size}x${size}`} href={icon} />
  ))
  }
- {!isProduction && <meta name="robots" content="noindex" />}
+ {(!isProduction || isArchive) && <meta name="robots" content="noindex" />}
  <meta name="generator" content={Astro.generator} />
  <title>{title}</title>
  <link rel="canonical" href={canonicalUrl}>

diff --git a/packages/ag-charts-website/src/pages/robots.txt.ts b/packages/ag-charts-website/src/pages/robots.txt.ts
@@ -19,6 +19,7 @@ Sitemap: ${pathJoin(SITE_URL, urlWithBaseUrl('/sitemap-index.xml'))}
 `;
 
 export async function GET(_context: APIContext) {
+ // NOTE: /archive is ignored in `ignorePaths` on production
  const disallowAll = !getIsDev() && !getIsProduction();
 
  const ignorePaths = await getSitemapIgnorePaths();

diff --git a/packages/ag-charts-website/src/utils/env.ts b/packages/ag-charts-website/src/utils/env.ts
@@ -1,5 +1,9 @@
-import { PRODUCTION_SITE_URL, SITE_URL, STAGING_SITE_URL } from '../constants';
+import { PRODUCTION_SITE_URL, SITE_BASE_URL, SITE_URL, STAGING_SITE_URL } from '../constants';
 
 export const getIsDev = () => import.meta.env?.DEV;
 export const getIsStaging = () => SITE_URL === STAGING_SITE_URL;
+/**
+ * Production environment, including archive
+ */
 export const getIsProduction = () => SITE_URL === PRODUCTION_SITE_URL;
+export const getIsArchive = () => SITE_URL === PRODUCTION_SITE_URL && SITE_BASE_URL.includes('archive');
diff --git a/packages/ag-charts-website/src/utils/sitemapPages.ts b/packages/ag-charts-website/src/utils/sitemapPages.ts
@@ -63,7 +63,11 @@ const getHiddenPages = async () => {
 };
 
 const getIgnoredPages = () => {
- return [urlWithBaseUrl('/404'), addTrailingSlash(urlWithBaseUrl('/gallery/examples'))];
+ return [
+ urlWithBaseUrl('/404'),
+ addTrailingSlash(urlWithBaseUrl('/gallery/examples')),
+ addTrailingSlash(urlWithBaseUrl('/archive')),
+ ];
 };
 
 export async function getSitemapIgnorePaths() {