feat(v2): Add themeConfig.noIndex option #3528 (#3573)

* feat(v2): Add themeConfig.noIndex option #3528

* feat(v2): Add themeConfig.noIndex optionthrough ssrTemplate

* feat(v2): Include themeConfig.noIndex in CLI migration v1 to v2

* feat(v2): Remove themeConfig.noIndex from env dev

packages/docusaurus-migrate/src/__tests__/__fixtures__/expectedSiteConfig.js

@@ -12,6 +12,7 @@ module.exports = {
   baseUrl: '/',
   organizationName: 'facebook',
   projectName: 'docusaurus',
+  noIndex: true,
   scripts: [
     'https://buttons.github.io/buttons.js',
     'https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js',

packages/docusaurus-migrate/src/__tests__/__fixtures__/sourceSiteConfig.js

@@ -21,7 +21,7 @@ const siteConfig = {
   organizationName: 'facebook',
   projectName: 'docusaurus',
   cname: 'docusaurus.io',
-  noIndex: false,
+  noIndex: true,
   users,
   editUrl: 'https://github.com/facebook/docusaurus/edit/master/docs/',
   headerLinks: [

packages/docusaurus-migrate/src/index.ts

@@ -296,6 +296,7 @@ export function createConfigFile({
     baseUrl: siteConfig.baseUrl ?? '',
     organizationName: siteConfig.organizationName,
     projectName: siteConfig.projectName,
+    noIndex: siteConfig.noIndex,
     scripts: siteConfig.scripts,
     stylesheets: siteConfig.stylesheets,
     favicon: siteConfig.favicon ?? '',

packages/docusaurus-migrate/src/types.ts

@@ -35,6 +35,7 @@ export interface VersionTwoConfig {
   url: string;
   organizationName?: string;
   projectName?: string;
+  noIndex?: boolean;
   githubHost?: string;
   onBrokenLinks: string;
   plugins: Array<[string, {[key: string]: any}]>;
@@ -94,7 +95,7 @@ export type VersionOneConfig = {
   defaultVersionShown?: string;
   organizationName?: string;
   projectName?: string;
-  noIndex?: string;
+  noIndex?: boolean;
   headerLinks?: Array<any>;
   headerIcon?: string;
   favicon?: string;

packages/docusaurus-types/src/index.d.ts

@@ -22,6 +22,7 @@ export interface DocusaurusConfig {
   url: string;
   onBrokenLinks: ReportingSeverity;
   onDuplicateRoutes: ReportingSeverity;
+  noIndex: boolean;
   organizationName?: string;
   projectName?: string;
   githubHost?: string;

packages/docusaurus/src/client/serverEntry.js

@@ -51,6 +51,7 @@ export default async function render(locals) {
     onLinksCollected,
     baseUrl,
     ssrTemplate,
+    noIndex,
   } = locals;
   const location = routesLocation[locals.path];
   await preload(routes, location);
@@ -101,6 +102,7 @@ export default async function render(locals) {
     metaAttributes,
     scripts,
     stylesheets,
+    noIndex,
     version: packageJson.version,
   });
 

packages/docusaurus/src/client/templates/ssr.html.template.js

@@ -12,6 +12,9 @@ module.exports = `
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="generator" content="Docusaurus v<%= it.version %>">
+    <%if (it.noIndex) { %>
+      <meta name="robots" content="noindex" />
+    <% } %>
     <%~ it.headTags %>
     <% it.metaAttributes.forEach((metaAttribute) => { %>
       <%~ metaAttribute %>

packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap

@@ -20,6 +20,7 @@ Object {
   "baseUrl": "/",
   "customFields": Object {},
   "favicon": "img/docusaurus.ico",
+  "noIndex": false,
   "onBrokenLinks": "throw",
   "onDuplicateRoutes": "warn",
   "organizationName": "endiliey",

packages/docusaurus/src/server/configValidation.ts

@@ -24,6 +24,7 @@ export const DEFAULT_CONFIG: Pick<
   | 'customFields'
   | 'themeConfig'
   | 'titleDelimiter'
+  | 'noIndex'
 > = {
   onBrokenLinks: 'throw',
   onDuplicateRoutes: 'warn',
@@ -33,6 +34,7 @@ export const DEFAULT_CONFIG: Pick<
   customFields: {},
   themeConfig: {},
   titleDelimiter: '|',
+  noIndex: false,
 };
 
 const PluginSchema = Joi.alternatives().try(
@@ -94,6 +96,7 @@ const ConfigSchema = Joi.object({
   clientModules: Joi.array().items(Joi.string()),
   tagline: Joi.string().allow(''),
   titleDelimiter: Joi.string().default('|'),
+  noIndex: Joi.bool().default(false),
 });
 
 // TODO move to @docusaurus/utils-validation

packages/docusaurus/src/webpack/server.ts

@@ -32,6 +32,7 @@ export default function createServerConfig({
     preBodyTags,
     postBodyTags,
     ssrTemplate,
+    siteConfig: {noIndex},
   } = props;
   const config = createBaseConfig(props, true, minify);
 
@@ -72,6 +73,7 @@ export default function createServerConfig({
           postBodyTags,
           onLinksCollected,
           ssrTemplate,
+          noIndex,
         },
         paths: ssgPaths,
       }),

website/docs/api/docusaurus.config.js.md

@@ -80,6 +80,20 @@ module.exports = {
 
 ## Optional fields
 
+### `noIndex`
+
+- Type: `boolean`
+
+This option adds `<meta name="robots" content="noindex">` in pages, to tell search engines to avoid indexing your site (more information [here](https://moz.com/learn/seo/robots-meta-directives)).
+
+Example:
+
+```js title="docusaurus.config.js"
+module.exports = {
+  noIndex: true, // Defaults to false
+};
+```
+
 ### `onBrokenLinks`
 
 - Type: `'ignore' | 'log' | 'warn' | 'error' | 'throw'`

website/docs/guides/migrating-from-v1-to-v2.md

@@ -440,7 +440,6 @@ The following fields are all deprecated, you may remove from your configuration
 - `markdownOptions` - We use MDX in v2 instead of Remarkable. Your markdown options have to be converted to Remark/Rehype plugins.
 - `markdownPlugins` - We use MDX in v2 instead of Remarkable. Your markdown plugins have to be converted to Remark/Rehype plugins.
 - `manifest`
-- `noIndex`
 - `onPageNav` - This is turned on by default now.
 - `separateCss` - It can imported in the same manner as `custom.css` mentioned above.
 - `scrollToTop`