3pp-mirror/docusaurus
1
0
Fork 0
mirror of https://github.com/facebook/docusaurus.git synced 2025-05-30 01:17:07 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix(sitemap): exclude pages with robots noindex from sitemap (#7143)

Browse source
This commit is contained in:
Joshua Chen 2022-04-14 17:31:09 +08:00 • committed by GitHub
parent 6306cbc266
commit 03516dc3a7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 105 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

16
packages/docusaurus-plugin-debug/src/index.ts
View file

@ -9,8 +9,6 @@ import type {LoadContext, Plugin} from '@docusaurus/types';
import {docuHash, normalizeUrl, posixPath} from '@docusaurus/utils';
import path from 'path';
export const routeBasePath = '__docusaurus/debug';
export default function pluginDebug({
siteConfig: {baseUrl},
generatedFilesDir,
@ -42,37 +40,37 @@ export default function pluginDebug({
// Home is config (duplicate for now)
addRoute({
path: normalizeUrl([baseUrl, routeBasePath]),
path: normalizeUrl([baseUrl, '__docusaurus/debug']),
component: '@theme/DebugConfig',
exact: true,
});
addRoute({
path: normalizeUrl([baseUrl, routeBasePath, 'config']),
path: normalizeUrl([baseUrl, '__docusaurus/debug/config']),
component: '@theme/DebugConfig',
exact: true,
});
addRoute({
path: normalizeUrl([baseUrl, routeBasePath, 'metadata']),
path: normalizeUrl([baseUrl, '__docusaurus/debug/metadata']),
component: '@theme/DebugSiteMetadata',
exact: true,
});
addRoute({
path: normalizeUrl([baseUrl, routeBasePath, 'registry']),
path: normalizeUrl([baseUrl, '__docusaurus/debug/registry']),
component: '@theme/DebugRegistry',
exact: true,
});
addRoute({
path: normalizeUrl([baseUrl, routeBasePath, 'routes']),
path: normalizeUrl([baseUrl, '__docusaurus/debug/routes']),
component: '@theme/DebugRoutes',
exact: true,
});
addRoute({
path: normalizeUrl([baseUrl, routeBasePath, 'content']),
path: normalizeUrl([baseUrl, '__docusaurus/debug/content']),
component: '@theme/DebugContent',
exact: true,
modules: {
@ -81,7 +79,7 @@ export default function pluginDebug({
});
addRoute({
path: normalizeUrl([baseUrl, routeBasePath, 'globalData']),
path: normalizeUrl([baseUrl, '__docusaurus/debug/globalData']),
component: '@theme/DebugGlobalData',
exact: true,
});

4
packages/docusaurus-plugin-debug/src/plugin-debug.d.ts vendored
View file

@ -7,10 +7,6 @@
/// <reference types="@docusaurus/module-type-aliases" />
declare module '@docusaurus/plugin-debug' {
export const routeBasePath: string;
}
declare module '@theme/DebugConfig' {
export default function DebugMetadata(): JSX.Element;
}

1
packages/docusaurus-plugin-debug/src/theme/DebugLayout/index.tsx
View file

@ -35,6 +35,7 @@ export default function DebugLayout({
<Head>
<html lang="en" />
<title>Docusaurus debug panel</title>
<meta name="robots" content="noindex" />
</Head>
<div>

35
packages/docusaurus-plugin-sitemap/src/__tests__/createSitemap.test.ts
View file

@ -5,6 +5,7 @@
* LICENSE file in the root directory of this source tree.
*/
import React from 'react';
import createSitemap from '../createSitemap';
import type {DocusaurusConfig} from '@docusaurus/types';
import {EnumChangefreq} from 'sitemap';
@ -16,6 +17,7 @@ describe('createSitemap', () => {
url: 'https://example.com',
} as DocusaurusConfig,
['/', '/test'],
{},
{
changefreq: EnumChangefreq.DAILY,
priority: 0.7,
@ -29,7 +31,7 @@ describe('createSitemap', () => {
it('empty site', () =>
expect(async () => {
await createSitemap({} as DocusaurusConfig, [], {});
await createSitemap({} as DocusaurusConfig, [], {}, {});
}).rejects.toThrow(
'URL in docusaurus.config.js cannot be empty/undefined.',
));
@ -40,6 +42,7 @@ describe('createSitemap', () => {
url: 'https://example.com',
} as DocusaurusConfig,
['/', '/404.html', '/my-page'],
{},
{
changefreq: EnumChangefreq.DAILY,
priority: 0.7,
@ -55,6 +58,7 @@ describe('createSitemap', () => {
url: 'https://example.com',
} as DocusaurusConfig,
['/', '/search/', '/tags/', '/search/foo', '/tags/foo/bar'],
{},
{
changefreq: EnumChangefreq.DAILY,
priority: 0.7,
@ -78,6 +82,7 @@ describe('createSitemap', () => {
trailingSlash: undefined,
} as DocusaurusConfig,
['/', '/test', '/nested/test', '/nested/test2/'],
{},
{
changefreq: EnumChangefreq.DAILY,
priority: 0.7,
@ -98,6 +103,7 @@ describe('createSitemap', () => {
trailingSlash: true,
} as DocusaurusConfig,
['/', '/test', '/nested/test', '/nested/test2/'],
{},
{
changefreq: EnumChangefreq.DAILY,
priority: 0.7,
@ -118,6 +124,7 @@ describe('createSitemap', () => {
trailingSlash: false,
} as DocusaurusConfig,
['/', '/test', '/nested/test', '/nested/test2/'],
{},
{
changefreq: EnumChangefreq.DAILY,
priority: 0.7,
@ -130,4 +137,30 @@ describe('createSitemap', () => {
expect(sitemap).toContain('<loc>https://example.com/nested/test</loc>');
expect(sitemap).toContain('<loc>https://example.com/nested/test2</loc>');
});
it('filters pages with noindex', async () => {
const sitemap = await createSitemap(
{
url: 'https://example.com',
trailingSlash: false,
} as DocusaurusConfig,
['/', '/noindex', '/nested/test', '/nested/test2/'],
{
'/noindex': {
meta: {
toComponent: () => [
React.createElement('meta', {name: 'robots', content: 'noindex'}),
],
},
},
},
{
changefreq: EnumChangefreq.DAILY,
priority: 0.7,
ignorePatterns: [],
},
);
expect(sitemap).not.toContain('/noindex');
});
});

20
packages/docusaurus-plugin-sitemap/src/createSitemap.ts
View file

@ -10,10 +10,13 @@ import type {PluginOptions} from '@docusaurus/plugin-sitemap';
import type {DocusaurusConfig} from '@docusaurus/types';
import {applyTrailingSlash} from '@docusaurus/utils-common';
import {createMatcher} from '@docusaurus/utils';
import type {HelmetServerState} from 'react-helmet-async';
import type {ReactElement} from 'react';
export default async function createSitemap(
siteConfig: DocusaurusConfig,
routesPaths: string[],
head: {[location: string]: HelmetServerState},
options: PluginOptions,
): Promise<string> {
const {url: hostname} = siteConfig;
@ -26,9 +29,20 @@ export default async function createSitemap(
const sitemapStream = new SitemapStream({hostname});
routesPaths
.filter((route) => !route.endsWith('404.html') && !ignoreMatcher(route))
.forEach((routePath) =>
function routeShouldBeIncluded(route: string) {
if (route.endsWith('404.html') || ignoreMatcher(route)) {
return false;
}
// https://github.com/staylor/react-helmet-async/pull/167
const meta = head[route]?.meta.toComponent() as unknown as
| ReactElement[]
| undefined;
return !meta?.some(
(tag) => tag.props.name === 'robots' && tag.props.content === 'noindex',
);
}
routesPaths.filter(routeShouldBeIncluded).forEach((routePath) =>
sitemapStream.write({
url: applyTrailingSlash(routePath, {
trailingSlash: siteConfig.trailingSlash,

3
packages/docusaurus-plugin-sitemap/src/index.ts
View file

@ -18,7 +18,7 @@ export default function pluginSitemap(
return {
name: 'docusaurus-plugin-sitemap',
async postBuild({siteConfig, routesPaths, outDir}) {
async postBuild({siteConfig, routesPaths, outDir, head}) {
if (siteConfig.noIndex) {
return;
}
@ -26,6 +26,7 @@ export default function pluginSitemap(
const generatedSitemap = await createSitemap(
siteConfig,
routesPaths,
head,
options,
);

12
packages/docusaurus-preset-classic/src/index.ts
View file

@ -5,7 +5,6 @@
* LICENSE file in the root directory of this source tree.
*/
import {routeBasePath as debugPluginRouteBasePath} from '@docusaurus/plugin-debug';
import type {
Preset,
LoadContext,
@ -29,7 +28,7 @@ export default function preset(
opts: Options = {},
): Preset {
const {siteConfig} = context;
const {themeConfig, baseUrl} = siteConfig;
const {themeConfig} = siteConfig;
const {algolia} = themeConfig as Partial<ThemeConfig>;
const isProd = process.env.NODE_ENV === 'production';
const {
@ -37,13 +36,12 @@ export default function preset(
docs,
blog,
pages,
sitemap = {},
sitemap,
theme,
googleAnalytics,
gtag,
...rest
} = opts;
const isDebugEnabled = debug || (debug === undefined && !isProd);
const themes: PluginConfig[] = [];
themes.push(makePluginConfig('@docusaurus/theme-classic', theme));
@ -76,17 +74,13 @@ export default function preset(
makePluginConfig('@docusaurus/plugin-google-analytics', googleAnalytics),
);
}
if (isDebugEnabled) {
if (debug || (debug === undefined && !isProd)) {
plugins.push(require.resolve('@docusaurus/plugin-debug'));
}
if (gtag) {
plugins.push(makePluginConfig('@docusaurus/plugin-google-gtag', gtag));
}
if (isProd && sitemap !== false) {
if (isDebugEnabled) {
sitemap.ignorePatterns ??= [];
sitemap.ignorePatterns.push(`${baseUrl}${debugPluginRouteBasePath}/**`);
}
plugins.push(makePluginConfig('@docusaurus/plugin-sitemap', sitemap));
}
if (Object.keys(rest).length > 0) {

1
packages/docusaurus-types/package.json
View file

@ -19,6 +19,7 @@
"commander": "^5.1.0",
"history": "^4.9.0",
"joi": "^17.6.0",
"react-helmet-async": "^1.2.3",
"utility-types": "^3.10.0",
"webpack": "^5.72.0",
"webpack-merge": "^5.8.0"

8
packages/docusaurus-types/src/index.d.ts vendored
View file

@ -10,6 +10,7 @@ import type {CustomizeRuleString} from 'webpack-merge/dist/types';
import type {CommanderStatic} from 'commander';
import type {ParsedUrlQueryInput} from 'querystring';
import type Joi from 'joi';
import type {HelmetServerState} from 'react-helmet-async';
import type {
DeepRequired,
Required as RequireKeys,
@ -319,7 +320,12 @@ export type Plugin<Content = unknown> = {
actions: PluginContentLoadedActions;
}) => Promise<void> | void;
routesLoaded?: (routes: RouteConfig[]) => void; // TODO remove soon, deprecated (alpha-60)
postBuild?: (props: Props & {content: Content}) => Promise<void> | void;
postBuild?: (
props: Props & {
content: Content;
head: {[location: string]: HelmetServerState};
},
) => Promise<void> | void;
// TODO refactor the configureWebpack API surface: use an object instead of
// multiple params (requires breaking change)
configureWebpack?: (

2
packages/docusaurus/src/client/serverEntry.tsx
View file

@ -70,6 +70,7 @@ async function doRender(locals: Locals & {path: string}) {
preBodyTags,
postBodyTags,
onLinksCollected,
onHeadTagsCollected,
baseUrl,
ssrTemplate,
noIndex,
@ -105,6 +106,7 @@ async function doRender(locals: Locals & {path: string}) {
helmet.link.toString(),
helmet.script.toString(),
];
onHeadTagsCollected(location, helmet);
const metaAttributes = metaStrings.filter(Boolean);
const {generatedFilesDir} = locals;

11
packages/docusaurus/src/commands/build.ts
View file

@ -27,6 +27,7 @@ import {
import CleanWebpackPlugin from '../webpack/plugins/CleanWebpackPlugin';
import {loadI18n} from '../server/i18n';
import {mapAsyncSequential} from '@docusaurus/utils';
import type {HelmetServerState} from 'react-helmet-async';
export async function build(
siteDir: string,
@ -149,12 +150,16 @@ async function buildLocale({
);
const allCollectedLinks: {[location: string]: string[]} = {};
const headTags: {[location: string]: HelmetServerState} = {};
let serverConfig: Configuration = await createServerConfig({
props,
onLinksCollected: (staticPagePath, links) => {
allCollectedLinks[staticPagePath] = links;
},
onHeadTagsCollected: (staticPagePath, tags) => {
headTags[staticPagePath] = tags;
},
});
if (staticDirectories.length > 0) {
@ -224,7 +229,11 @@ async function buildLocale({
if (!plugin.postBuild) {
return;
}
await plugin.postBuild({...props, content: plugin.content});
await plugin.postBuild({
...props,
head: headTags,
content: plugin.content,
});
}),
);

6
packages/docusaurus/src/deps.d.ts vendored
View file

@ -35,6 +35,8 @@ declare module 'react-loadable-ssr-addon-v5-slorber' {
}
declare module '@slorber/static-site-generator-webpack-plugin' {
import type {HelmetServerState} from 'react-helmet-async';
export type Locals = {
routesLocation: {[filePath: string]: string};
generatedFilesDir: string;
@ -42,6 +44,10 @@ declare module '@slorber/static-site-generator-webpack-plugin' {
preBodyTags: string;
postBodyTags: string;
onLinksCollected: (staticPagePath: string, links: string[]) => void;
onHeadTagsCollected: (
staticPagePath: string,
tags: HelmetServerState,
) => void;
baseUrl: string;
ssrTemplate: string;
noIndex: boolean;

11
packages/docusaurus/src/webpack/server.ts
View file

@ -17,14 +17,16 @@ import {NODE_MAJOR_VERSION, NODE_MINOR_VERSION} from '@docusaurus/utils';
import ssrDefaultTemplate from './templates/ssr.html.template';
// Forked for Docusaurus: https://github.com/slorber/static-site-generator-webpack-plugin
import StaticSiteGeneratorPlugin from '@slorber/static-site-generator-webpack-plugin';
import StaticSiteGeneratorPlugin, {
type Locals,
} from '@slorber/static-site-generator-webpack-plugin';
export default async function createServerConfig({
props,
onLinksCollected = () => {},
}: {
onLinksCollected,
onHeadTagsCollected,
}: Pick<Locals, 'onLinksCollected' | 'onHeadTagsCollected'> & {
props: Props;
onLinksCollected?: (staticPagePath: string, links: string[]) => void;
}): Promise<Configuration> {
const {
baseUrl,
@ -73,6 +75,7 @@ export default async function createServerConfig({
preBodyTags,
postBodyTags,
onLinksCollected,
onHeadTagsCollected,
ssrTemplate: ssrTemplate ?? ssrDefaultTemplate,
noIndex,
},