feat(v2): generalize usage of _ prefix convention to exclude content files/folders (#5173)

* create a swizzleWarning partial for shared text

* Generalize usage of _ prefix convention to exclude content files/folders

* add api doc

* MDX loader should not expect metadata/frontmatter on MDX partial files

packages/docusaurus-mdx-loader/src/index.js

@@ -24,6 +24,19 @@ const DEFAULT_OPTIONS = {
   remarkPlugins: [unwrapMdxCodeBlocks, emoji, headings, toc],
 };
 
+// When this throws, it generally means that there's no metadata file associated with this MDX document
+// It can happen when using MDX partials (usually starting with _)
+// That's why it's important to provide the "isMDXPartial" function in config
+async function readMetadataPath(metadataPath) {
+  try {
+    return await readFile(metadataPath, 'utf8');
+  } catch (e) {
+    throw new Error(
+      `MDX loader can't read MDX metadata file for path ${metadataPath}. Maybe the isMDXPartial option function was not provided?`,
+    );
+  }
+}
+
 module.exports = async function docusaurusMdxLoader(fileString) {
   const callback = this.async();
 
@@ -37,19 +50,15 @@ module.exports = async function docusaurusMdxLoader(fileString) {
 
   const hasFrontMatter = Object.keys(frontMatter).length > 0;
 
+  const filePath = this.resourcePath;
+
   const options = {
     ...reqOptions,
     remarkPlugins: [
       ...(reqOptions.beforeDefaultRemarkPlugins || []),
       ...DEFAULT_OPTIONS.remarkPlugins,
-      [
+      [transformImage, {staticDir: reqOptions.staticDir, filePath}],
-        transformImage,
+      [transformLinks, {staticDir: reqOptions.staticDir, filePath}],
-        {staticDir: reqOptions.staticDir, filePath: this.resourcePath},
-      ],
-      [
-        transformLinks,
-        {staticDir: reqOptions.staticDir, filePath: this.resourcePath},
-      ],
       ...(reqOptions.remarkPlugins || []),
     ],
     rehypePlugins: [
@@ -58,7 +67,7 @@ module.exports = async function docusaurusMdxLoader(fileString) {
 
       ...(reqOptions.rehypePlugins || []),
     ],
-    filepath: this.resourcePath,
+    filepath: filePath,
   };
   let result;
 
@@ -74,27 +83,33 @@ module.exports = async function docusaurusMdxLoader(fileString) {
     contentTitle,
   )};`;
 
+  // MDX partials are MDX files starting with _ or in a folder starting with _
+  // Partial are not expected to have an associated metadata file or frontmatter
+  const isMDXPartial = options.isMDXPartial
+    ? options.isMDXPartial(filePath)
+    : false;
+
+  if (isMDXPartial && hasFrontMatter) {
+    return callback(
+      new Error(`MDX partial should not contain FrontMatter: ${filePath}`),
+    );
+  }
+
+  if (!isMDXPartial) {
     // Read metadata for this MDX and export it.
     if (options.metadataPath && typeof options.metadataPath === 'function') {
-    const metadataPath = options.metadataPath(this.resourcePath);
+      const metadataPath = options.metadataPath(filePath);
 
       if (metadataPath) {
+        const metadata = await readMetadataPath(metadataPath);
+        exportStr += `\nexport const metadata = ${metadata};`;
         // Add as dependency of this loader result so that we can
         // recompile if metadata is changed.
         this.addDependency(metadataPath);
-      const metadata = await readFile(metadataPath, 'utf8');
+      }
-      exportStr += `\nexport const metadata = ${metadata};`;
     }
   }
 
-  if (
-    options.forbidFrontMatter &&
-    typeof options.forbidFrontMatter === 'function'
-  ) {
-    if (options.forbidFrontMatter(this.resourcePath) && hasFrontMatter) {
-      return callback(new Error(`Front matter is forbidden in this file`));
-    }
-  }
   const code = `
   import React from 'react';
   import { mdx } from '@mdx-js/react';

packages/docusaurus-plugin-content-blog/src/__tests__/__fixtures__/website/blog/_partials/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-blog/src/__tests__/__fixtures__/website/blog/_partials/subfolder/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-blog/src/__tests__/__fixtures__/website/blog/_somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with \_:
+
+It should be excluded by default

packages/docusaurus-plugin-content-blog/src/__tests__/__snapshots__/generateBlogFeed.test.ts.snap

@@ -7,12 +7,26 @@ exports[`blogFeed atom shows feed item for each post 1`] = `
 <feed xmlns=\\"http://www.w3.org/2005/Atom\\">
     <id>https://docusaurus.io/myBaseUrl/blog</id>
     <title>Hello Blog</title>
-    <updated>2020-02-27T00:00:00.000Z</updated>
+    <updated>2020-08-16T00:00:00.000Z</updated>
     <generator>https://github.com/jpmonette/feed</generator>
     <link rel=\\"alternate\\" href=\\"https://docusaurus.io/myBaseUrl/blog\\"/>
     <subtitle>Hello Blog</subtitle>
     <icon>https://docusaurus.io/myBaseUrl/image/favicon.ico</icon>
     <rights>Copyright</rights>
+    <entry>
+        <title type=\\"html\\"><![CDATA[Complex Slug]]></title>
+        <id>/hey/my super path/héllô</id>
+        <link href=\\"https://docusaurus.io/myBaseUrl/blog/hey/my super path/héllô\\"/>
+        <updated>2020-08-16T00:00:00.000Z</updated>
+        <summary type=\\"html\\"><![CDATA[complex url slug]]></summary>
+    </entry>
+    <entry>
+        <title type=\\"html\\"><![CDATA[Simple Slug]]></title>
+        <id>/simple/slug</id>
+        <link href=\\"https://docusaurus.io/myBaseUrl/blog/simple/slug\\"/>
+        <updated>2020-08-15T00:00:00.000Z</updated>
+        <summary type=\\"html\\"><![CDATA[simple url slug]]></summary>
+    </entry>
     <entry>
         <title type=\\"html\\"><![CDATA[draft]]></title>
         <id>draft</id>
@@ -20,6 +34,12 @@ exports[`blogFeed atom shows feed item for each post 1`] = `
         <updated>2020-02-27T00:00:00.000Z</updated>
         <summary type=\\"html\\"><![CDATA[this post should not be published yet]]></summary>
     </entry>
+    <entry>
+        <title type=\\"html\\"><![CDATA[some heading]]></title>
+        <id>some heading</id>
+        <link href=\\"https://docusaurus.io/myBaseUrl/blog/heading-as-title\\"/>
+        <updated>2019-01-02T00:00:00.000Z</updated>
+    </entry>
     <entry>
         <title type=\\"html\\"><![CDATA[date-matter]]></title>
         <id>date-matter</id>
@@ -46,10 +66,24 @@ exports[`blogFeed rss shows feed item for each post 1`] = `
         <title>Hello Blog</title>
         <link>https://docusaurus.io/myBaseUrl/blog</link>
         <description>Hello Blog</description>
-        <lastBuildDate>Thu, 27 Feb 2020 00:00:00 GMT</lastBuildDate>
+        <lastBuildDate>Sun, 16 Aug 2020 00:00:00 GMT</lastBuildDate>
         <docs>https://validator.w3.org/feed/docs/rss2.html</docs>
         <generator>https://github.com/jpmonette/feed</generator>
         <copyright>Copyright</copyright>
+        <item>
+            <title><![CDATA[Complex Slug]]></title>
+            <link>https://docusaurus.io/myBaseUrl/blog/hey/my super path/héllô</link>
+            <guid>/hey/my super path/héllô</guid>
+            <pubDate>Sun, 16 Aug 2020 00:00:00 GMT</pubDate>
+            <description><![CDATA[complex url slug]]></description>
+        </item>
+        <item>
+            <title><![CDATA[Simple Slug]]></title>
+            <link>https://docusaurus.io/myBaseUrl/blog/simple/slug</link>
+            <guid>/simple/slug</guid>
+            <pubDate>Sat, 15 Aug 2020 00:00:00 GMT</pubDate>
+            <description><![CDATA[simple url slug]]></description>
+        </item>
         <item>
             <title><![CDATA[draft]]></title>
             <link>https://docusaurus.io/myBaseUrl/blog/draft</link>
@@ -57,6 +91,12 @@ exports[`blogFeed rss shows feed item for each post 1`] = `
             <pubDate>Thu, 27 Feb 2020 00:00:00 GMT</pubDate>
             <description><![CDATA[this post should not be published yet]]></description>
         </item>
+        <item>
+            <title><![CDATA[some heading]]></title>
+            <link>https://docusaurus.io/myBaseUrl/blog/heading-as-title</link>
+            <guid>some heading</guid>
+            <pubDate>Wed, 02 Jan 2019 00:00:00 GMT</pubDate>
+        </item>
         <item>
             <title><![CDATA[date-matter]]></title>
             <link>https://docusaurus.io/myBaseUrl/blog/date-matter</link>

packages/docusaurus-plugin-content-blog/src/__tests__/generateBlogFeed.test.ts

@@ -9,6 +9,7 @@ import path from 'path';
 import {generateBlogFeed} from '../blogUtils';
 import {LoadContext, I18n} from '@docusaurus/types';
 import {PluginOptions, BlogContentPaths} from '../types';
+import {DEFAULT_OPTIONS} from '../pluginOptionSchema';
 
 const DefaultI18N: I18n = {
   currentLocale: 'en',
@@ -84,7 +85,8 @@ describe('blogFeed', () => {
           {
             path: 'blog',
             routeBasePath: 'blog',
-            include: ['*r*.md', '*.mdx'], // skip no-date.md - it won't play nice with snapshots
+            include: DEFAULT_OPTIONS.include,
+            exclude: DEFAULT_OPTIONS.exclude,
             feedOptions: {
               type: [feedType],
               copyright: 'Copyright',

packages/docusaurus-plugin-content-blog/src/blogUtils.ts

@@ -6,7 +6,6 @@
  */
 
 import fs from 'fs-extra';
-import globby from 'globby';
 import chalk from 'chalk';
 import path from 'path';
 import readingTime from 'reading-time';
@@ -27,6 +26,7 @@ import {
   getFolderContainingFile,
   posixPath,
   replaceMarkdownLinks,
+  Globby,
 } from '@docusaurus/utils';
 import {LoadContext} from '@docusaurus/types';
 import {validateBlogPostFrontMatter} from './blogFrontMatter';
@@ -127,6 +127,7 @@ export async function generateBlogPosts(
 ): Promise<BlogPost[]> {
   const {
     include,
+    exclude,
     routeBasePath,
     truncateMarker,
     showReadingTime,
@@ -138,8 +139,9 @@ export async function generateBlogPosts(
   }
 
   const {baseUrl = ''} = siteConfig;
-  const blogSourceFiles = await globby(include, {
+  const blogSourceFiles = await Globby(include, {
     cwd: contentPaths.contentPath,
+    ignore: exclude,
   });
 
   const blogPosts: BlogPost[] = [];

packages/docusaurus-plugin-content-blog/src/index.ts

@@ -16,6 +16,7 @@ import {
   reportMessage,
   posixPath,
   addTrailingPathSeparator,
+  createMatcher,
 } from '@docusaurus/utils';
 import {
   STATIC_DIR_NAME,
@@ -459,9 +460,10 @@ export default function pluginContentBlog(
                     beforeDefaultRemarkPlugins,
                     beforeDefaultRehypePlugins,
                     staticDir: path.join(siteDir, STATIC_DIR_NAME),
+                    isMDXPartial: createMatcher(options.exclude),
+                    metadataPath: (mdxPath: string) => {
                       // Note that metadataPath must be the same/in-sync as
                       // the path from createData for each MDX.
-                    metadataPath: (mdxPath: string) => {
                       const aliasedPath = aliasedSitePath(mdxPath, siteDir);
                       return path.join(
                         dataDir,

packages/docusaurus-plugin-content-blog/src/pluginOptionSchema.ts

@@ -12,6 +12,7 @@ import {
   AdmonitionsSchema,
   URISchema,
 } from '@docusaurus/utils-validation';
+import {GlobExcludeDefault} from '@docusaurus/utils';
 
 export const DEFAULT_OPTIONS = {
   feedOptions: {type: ['rss', 'atom']},
@@ -32,6 +33,7 @@ export const DEFAULT_OPTIONS = {
   blogSidebarTitle: 'Recent posts',
   postsPerPage: 10,
   include: ['*.md', '*.mdx'],
+  exclude: GlobExcludeDefault,
   routeBasePath: 'blog',
   path: 'blog',
   editLocalizedFiles: false,
@@ -44,6 +46,7 @@ export const PluginOptionSchema = Joi.object({
     // .allow('')
     .default(DEFAULT_OPTIONS.routeBasePath),
   include: Joi.array().items(Joi.string()).default(DEFAULT_OPTIONS.include),
+  exclude: Joi.array().items(Joi.string()).default(DEFAULT_OPTIONS.exclude),
   postsPerPage: Joi.number()
     .integer()
     .min(1)

packages/docusaurus-plugin-content-blog/src/types.ts

@@ -39,6 +39,7 @@ export interface PluginOptions extends RemarkAndRehypePluginOptions {
   path: string;
   routeBasePath: string;
   include: string[];
+  exclude: string[];
   postsPerPage: number;
   blogListComponent: string;
   blogPostComponent: string;

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docs/_partials/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docs/_partials/subfolder/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docs/_somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with \_:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/versioned-site/versioned_docs/version-1.0.1/_partials/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/versioned-site/versioned_docs/version-1.0.1/_partials/subfolder/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/versioned-site/versioned_docs/version-1.0.1/_somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with \_:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/options.test.ts

@@ -12,6 +12,7 @@ import {
   DefaultNumberPrefixParser,
   DisabledNumberPrefixParser,
 } from '../numberPrefix';
+import {GlobExcludeDefault} from '@docusaurus/utils';
 
 // the type of remark/rehype plugins is function
 const markdownPluginsFunctionStub = () => {};
@@ -30,6 +31,7 @@ describe('normalizeDocsPluginOptions', () => {
       routeBasePath: 'my-docs', // URL Route.
       homePageId: 'home', // Document id for docs home page.
       include: ['**/*.{md,mdx}'], // Extensions to include.
+      exclude: GlobExcludeDefault,
       sidebarPath: 'my-sidebar', // Path to sidebar configuration for showing a list of markdown pages.
       sidebarItemsGenerator: DefaultSidebarItemsGenerator,
       numberPrefixParser: DefaultNumberPrefixParser,

packages/docusaurus-plugin-content-docs/src/docs.ts

@@ -14,6 +14,7 @@ import {
   normalizeUrl,
   parseMarkdownString,
   posixPath,
+  Globby,
 } from '@docusaurus/utils';
 import {LoadContext} from '@docusaurus/types';
 
@@ -28,7 +29,6 @@ import {
 } from './types';
 import getSlug from './slug';
 import {CURRENT_VERSION_NAME} from './constants';
-import globby from 'globby';
 import {getDocsDirPaths} from './versions';
 import {stripPathNumberPrefixes} from './numberPrefix';
 import {validateDocFrontMatter} from './docFrontMatter';
@@ -92,11 +92,12 @@ export async function readVersionDocs(
   versionMetadata: VersionMetadata,
   options: Pick<
     PluginOptions,
-    'include' | 'showLastUpdateAuthor' | 'showLastUpdateTime'
+    'include' | 'exclude' | 'showLastUpdateAuthor' | 'showLastUpdateTime'
   >,
 ): Promise<DocFile[]> {
-  const sources = await globby(options.include, {
+  const sources = await Globby(options.include, {
     cwd: versionMetadata.contentPath,
+    ignore: options.exclude,
   });
   return Promise.all(
     sources.map((source) => readDocFile(versionMetadata, source, options)),

packages/docusaurus-plugin-content-docs/src/index.ts

@@ -18,6 +18,7 @@ import {
   reportMessage,
   posixPath,
   addTrailingPathSeparator,
+  createMatcher,
 } from '@docusaurus/utils';
 import {LoadContext, Plugin, RouteConfig} from '@docusaurus/types';
 import {loadSidebars, createSidebarsUtils, processSidebars} from './sidebars';
@@ -408,6 +409,7 @@ export default function pluginContentDocs(
                 beforeDefaultRehypePlugins,
                 beforeDefaultRemarkPlugins,
                 staticDir: path.join(siteDir, STATIC_DIR_NAME),
+                isMDXPartial: createMatcher(options.exclude),
                 metadataPath: (mdxPath: string) => {
                   // Note that metadataPath must be the same/in-sync as
                   // the path from createData for each MDX.

packages/docusaurus-plugin-content-docs/src/options.ts

@@ -12,6 +12,8 @@ import {
   AdmonitionsSchema,
   URISchema,
 } from '@docusaurus/utils-validation';
+import {GlobExcludeDefault} from '@docusaurus/utils';
+
 import {OptionValidationContext, ValidationResult} from '@docusaurus/types';
 import chalk from 'chalk';
 import admonitions from 'remark-admonitions';
@@ -26,6 +28,7 @@ export const DEFAULT_OPTIONS: Omit<PluginOptions, 'id' | 'sidebarPath'> = {
   routeBasePath: 'docs', // URL Route.
   homePageId: undefined, // TODO remove soon, deprecated
   include: ['**/*.{md,mdx}'], // Extensions to include.
+  exclude: GlobExcludeDefault,
   sidebarItemsGenerator: DefaultSidebarItemsGenerator,
   numberPrefixParser: DefaultNumberPrefixParser,
   docLayoutComponent: '@theme/DocPage',
@@ -66,6 +69,7 @@ export const OptionsSchema = Joi.object({
     .default(DEFAULT_OPTIONS.routeBasePath),
   homePageId: Joi.string().optional(),
   include: Joi.array().items(Joi.string()).default(DEFAULT_OPTIONS.include),
+  exclude: Joi.array().items(Joi.string()).default(DEFAULT_OPTIONS.exclude),
   sidebarPath: Joi.alternatives().try(
     Joi.boolean().invalid(true),
     Joi.string(),

packages/docusaurus-plugin-content-docs/src/types.ts

@@ -81,6 +81,7 @@ export type PluginOptions = MetadataOptions &
   RemarkAndRehypePluginOptions & {
     id: string;
     include: string[];
+    exclude: string[];
     docLayoutComponent: string;
     docItemComponent: string;
     admonitions: Record<string, unknown>;

packages/docusaurus-plugin-content-pages/package.json

@@ -25,9 +25,7 @@
     "@docusaurus/utils-validation": "2.0.0-beta.3",
     "globby": "^11.0.2",
     "lodash": "^4.17.20",
-    "minimatch": "^3.0.4",
     "remark-admonitions": "^1.2.1",
-    "slash": "^3.0.0",
     "tslib": "^2.1.0",
     "webpack": "^5.40.0"
   },

packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/src/pages/hello/_partials/partial.mdx

@@ -0,0 +1 @@
+# ignored

packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/src/pages/hello/_partials/subfolder/partial.mdx

@@ -0,0 +1 @@
+# ignored

packages/docusaurus-plugin-content-pages/src/index.ts

@@ -5,11 +5,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import globby from 'globby';
 import fs from 'fs';
 import path from 'path';
-import minimatch from 'minimatch';
-import slash from 'slash';
 import {
   encodePath,
   fileToPath,
@@ -18,6 +15,8 @@ import {
   getPluginI18nPath,
   getFolderContainingFile,
   addTrailingPathSeparator,
+  Globby,
+  createMatcher,
 } from '@docusaurus/utils';
 import {
   LoadContext,
@@ -81,11 +80,6 @@ export default function pluginContentPages(
   );
   const dataDir = path.join(pluginDataDirRoot, options.id ?? DEFAULT_PLUGIN_ID);
 
-  const excludeRegex = new RegExp(
-    options.exclude
-      .map((pattern) => minimatch.makeRe(pattern).source)
-      .join('|'),
-  );
   return {
     name: 'docusaurus-plugin-content-pages',
 
@@ -116,7 +110,7 @@ export default function pluginContentPages(
       }
 
       const {baseUrl} = siteConfig;
-      const pagesFiles = await globby(include, {
+      const pagesFiles = await Globby(include, {
         cwd: contentPaths.contentPath,
         ignore: options.exclude,
       });
@@ -223,20 +217,16 @@ export default function pluginContentPages(
                     beforeDefaultRehypePlugins,
                     beforeDefaultRemarkPlugins,
                     staticDir: path.join(siteDir, STATIC_DIR_NAME),
+                    isMDXPartial: createMatcher(options.exclude),
+                    metadataPath: (mdxPath: string) => {
                       // Note that metadataPath must be the same/in-sync as
                       // the path from createData for each MDX.
-                    metadataPath: (mdxPath: string) => {
-                      if (excludeRegex.test(slash(mdxPath))) {
-                        return null;
-                      }
                       const aliasedSource = aliasedSitePath(mdxPath, siteDir);
                       return path.join(
                         dataDir,
                         `${docuHash(aliasedSource)}.json`,
                       );
                     },
-                    forbidFrontMatter: (mdxPath: string) =>
-                      excludeRegex.test(slash(mdxPath)),
                   },
                 },
                 {

packages/docusaurus-plugin-content-pages/src/pluginOptionSchema.ts

@@ -11,22 +11,19 @@ import {
   RehypePluginsSchema,
   AdmonitionsSchema,
 } from '@docusaurus/utils-validation';
+import {GlobExcludeDefault} from '@docusaurus/utils';
 
 export const DEFAULT_OPTIONS: PluginOptions = {
   path: 'src/pages', // Path to data on filesystem, relative to site dir.
   routeBasePath: '/', // URL Route.
   include: ['**/*.{js,jsx,ts,tsx,md,mdx}'], // Extensions to include.
+  exclude: GlobExcludeDefault,
   mdxPageComponent: '@theme/MDXPage',
   remarkPlugins: [],
   rehypePlugins: [],
   beforeDefaultRehypePlugins: [],
   beforeDefaultRemarkPlugins: [],
   admonitions: {},
-  exclude: [
-    '**/_*.{js,jsx,ts,tsx,md,mdx}',
-    '**/*.test.{js,ts}',
-    '**/__tests__/**',
-  ],
 };
 
 export const PluginOptionSchema = Joi.object({

packages/docusaurus-utils/package.json

@@ -25,6 +25,8 @@
     "fs-extra": "^10.0.0",
     "gray-matter": "^4.0.3",
     "lodash": "^4.17.20",
+    "globby": "^11.0.4",
+    "micromatch": "^4.0.4",
     "resolve-pathname": "^3.0.0",
     "tslib": "^2.2.0"
   },
@@ -33,6 +35,7 @@
   },
   "devDependencies": {
     "@types/dedent": "^0.7.0",
+    "@types/micromatch": "^4.0.2",
     "dedent": "^0.7.0"
   }
 }

packages/docusaurus-utils/src/globUtils.ts

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Globby/Micromatch are the 2 libs we use in Docusaurus consistently
+
+export {default as Globby} from 'globby';
+
+import Micromatch from 'micromatch'; // Note: Micromatch is used by Globby
+
+// The default patterns we ignore when globbing
+// using _ prefix for exclusion by convention
+export const GlobExcludeDefault = [
+  // Ignore files starting with _
+  '**/_*.{js,jsx,ts,tsx,md,mdx}',
+
+  // Ignore folders starting with _ (including folder content)
+  '**/_*/**',
+
+  // Ignore tests
+  '**/*.test.{js,jsx,ts,tsx}',
+  '**/__tests__/**',
+];
+
+type Matcher = (str: string) => boolean;
+
+export function createMatcher(patterns: string[]): Matcher {
+  const regexp = new RegExp(
+    patterns.map((pattern) => Micromatch.makeRe(pattern).source).join('|'),
+  );
+  return (str) => regexp.test(str);
+}

packages/docusaurus-utils/src/index.ts

@@ -31,6 +31,7 @@ export * from './markdownParser';
 export * from './markdownLinks';
 export * from './escapePath';
 export {md5Hash, simpleHash, docuHash} from './hashUtils';
+export {Globby, GlobExcludeDefault, createMatcher} from './globUtils';
 
 const fileHash = new Map();
 export async function generate(

packages/docusaurus/src/server/themes/alias.ts

@@ -5,10 +5,9 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import globby from 'globby';
 import fs from 'fs-extra';
 import path from 'path';
-import {fileToPath, posixPath, normalizeUrl} from '@docusaurus/utils';
+import {fileToPath, posixPath, normalizeUrl, Globby} from '@docusaurus/utils';
 import {ThemeAliases} from '@docusaurus/types';
 import {sortBy} from 'lodash';
 
@@ -21,7 +20,7 @@ export default function themeAlias(
     return {};
   }
 
-  const themeComponentFiles = globby.sync(['**/*.{js,jsx,ts,tsx}'], {
+  const themeComponentFiles = Globby.sync(['**/*.{js,jsx,ts,tsx}'], {
     cwd: themePath,
   });
 

packages/docusaurus/src/server/utils.ts

@@ -6,9 +6,8 @@
  */
 import {flatMap} from 'lodash';
 import {RouteConfig} from '@docusaurus/types';
-import globby from 'globby';
 import nodePath from 'path';
-import {posixPath} from '@docusaurus/utils';
+import {posixPath, Globby} from '@docusaurus/utils';
 
 // Recursively get the final routes (routes with no subroutes)
 export function getAllFinalRoutes(routeConfig: RouteConfig[]): RouteConfig[] {
@@ -22,7 +21,7 @@ export function getAllFinalRoutes(routeConfig: RouteConfig[]): RouteConfig[] {
 // See https://github.com/facebook/docusaurus/pull/4222#issuecomment-795517329
 export async function safeGlobby(
   patterns: string[],
-  options?: globby.GlobbyOptions,
+  options?: Globby.GlobbyOptions,
 ): Promise<string[]> {
   // Required for Windows support, as paths using \ should not be used by globby
   // (also using the windows hard drive prefix like c: is not a good idea)
@@ -30,5 +29,5 @@ export async function safeGlobby(
     posixPath(nodePath.relative(process.cwd(), dirPath)),
   );
 
-  return globby(globPaths, options);
+  return Globby(globPaths, options);
 }

website/docs/_partials/swizzleWarning.mdx

@@ -0,0 +1,5 @@
+:::caution
+
+We discourage swizzling of components during the Docusaurus 2 beta phase. The theme components APIs are likely to evolve and have breaking changes. If possible, stick with the default appearance for now.
+
+:::

website/docs/api/plugins/plugin-content-blog.md

@@ -72,6 +72,15 @@ module.exports = {
          */
         routeBasePath: 'blog',
         include: ['*.md', '*.mdx'],
+        /**
+         * No route will be created for matching files
+         */
+        exclude: [
+          '**/_*.{js,jsx,ts,tsx,md,mdx}',
+          '**/_*/**',
+          '**/*.test.{js,jsx,ts,tsx}',
+          '**/__tests__/**',
+        ],
         postsPerPage: 10,
         /**
          * Theme components used by the blog pages.

website/docs/api/plugins/plugin-content-docs.md

@@ -69,6 +69,15 @@ module.exports = {
          */
         routeBasePath: 'docs',
         include: ['**/*.md', '**/*.mdx'], // Extensions to include.
+        /**
+         * No route will be created for matching files
+         */
+        exclude: [
+          '**/_*.{js,jsx,ts,tsx,md,mdx}',
+          '**/_*/**',
+          '**/*.test.{js,jsx,ts,tsx}',
+          '**/__tests__/**',
+        ],
         /**
          * Path to sidebar configuration for showing a list of markdown pages.
          */

website/docs/api/plugins/plugin-content-pages.md

@@ -39,11 +39,12 @@ module.exports = {
         routeBasePath: '',
         include: ['**/*.{js,jsx,ts,tsx,md,mdx}'],
         /**
-         * No Route will be created for matching files
+         * No route will be created for matching files
          */
         exclude: [
           '**/_*.{js,jsx,ts,tsx,md,mdx}',
-          '**/*.test.{js,ts}',
+          '**/_*/**',
+          '**/*.test.{js,jsx,ts,tsx}',
           '**/__tests__/**',
         ],
         /**

website/docs/cli.md

@@ -93,11 +93,11 @@ For advanced minification of CSS bundle, we use the [advanced cssnano preset](ht
 
 ### `docusaurus swizzle [siteDir]` {#docusaurus-swizzle-sitedir}
 
-:::caution
+```mdx-code-block
+import SwizzleWarning from "./_partials/swizzleWarning.mdx"
 
-We discourage swizzling of components during the Docusaurus 2 beta phase. The theme components APIs are likely to evolve and have breaking changes. If possible, stick with the default appearance for now.
+<SwizzleWarning/>
-
+```
-:::
 
 Change any Docusaurus theme components to your liking with `npm run swizzle`.
 

website/docs/using-themes.md

@@ -98,11 +98,11 @@ Use this component to render React Context providers and global stateful logic.
 
 ## Swizzling theme components {#swizzling-theme-components}
 
-:::caution
+```mdx-code-block
+import SwizzleWarning from "./_partials/swizzleWarning.mdx"
 
-We discourage swizzling of components during the Docusaurus 2 beta phase. The theme components APIs are likely to evolve and have breaking changes. If possible, stick with the default appearance for now.
+<SwizzleWarning/>
-
+```
-:::
 
 Docusaurus Themes' components are designed to be replaceable. To make it easier for you, we created a command for you to replace theme components called `swizzle`.
 

website/dogfooding/second-blog/2020-08-03-second-blog-intro.md

@@ -17,3 +17,9 @@ Did you know you can use multiple instances of the same plugin?
 Using twice the blog plugin permits you to create more than one blog on the same Docusaurus website!
 
 :::
+
+```mdx-code-block
+import Partial from "./_partial.mdx"
+
+<Partial />
+```

website/dogfooding/second-blog/_partial.mdx

@@ -0,0 +1 @@
+This is a blog partial

yarn.lock

@@ -3545,6 +3545,11 @@
     "@types/connect" "*"
     "@types/node" "*"
 
+"@types/braces@*":
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/@types/braces/-/braces-3.0.1.tgz#5a284d193cfc61abb2e5a50d36ebbc50d942a32b"
+  integrity sha512-+euflG6ygo4bn0JHtn4pYqcXwRtLvElQ7/nnjDu7iYG56H0+OhCd7d6Ug0IE3WcFpZozBKW2+80FUbv5QGk5AQ==
+
 "@types/clean-css@*":
   version "4.2.4"
   resolved "https://registry.yarnpkg.com/@types/clean-css/-/clean-css-4.2.4.tgz#4fe4705c384e6ec9ee8454bc3d49089f38dc038a"
@@ -3829,6 +3834,13 @@
   dependencies:
     "@types/unist" "*"
 
+"@types/micromatch@^4.0.2":
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/@types/micromatch/-/micromatch-4.0.2.tgz#ce29c8b166a73bf980a5727b1e4a4d099965151d"
+  integrity sha512-oqXqVb0ci19GtH0vOA/U2TmHTcRY9kuZl4mqUxe0QmJAlIW13kzhuK5pi1i9+ngav8FjpSb9FVS/GE00GLX1VA==
+  dependencies:
+    "@types/braces" "*"
+
 "@types/mime@^1":
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
@@ -9587,10 +9599,10 @@ globby@^10.0.1, globby@^10.0.2:
     merge2 "^1.2.3"
     slash "^3.0.0"
 
-globby@^11.0.1, globby@^11.0.2, globby@^11.0.3:
+globby@^11.0.1, globby@^11.0.2, globby@^11.0.3, globby@^11.0.4:
-  version "11.0.3"
+  version "11.0.4"
-  resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.3.tgz#9b1f0cb523e171dd1ad8c7b2a9fb4b644b9593cb"
+  resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.4.tgz#2cbaff77c2f2a62e71e9b2813a67b97a3a3001a5"
-  integrity sha512-ffdmosjA807y7+lA1NM0jELARVmYul/715xiILEjo3hBLPTcirgQNnXECn5g3mtR8TOLCVbkfua1Hpen25/Xcg==
+  integrity sha512-9O4MVG9ioZJ08ffbcyVYyLOJLk5JQ688pJ4eMGLpdWLHq/Wr1D9BlriLQyL0E+jbkuePVZXYFj47QM/v093wHg==
   dependencies:
     array-union "^2.1.0"
     dir-glob "^3.0.1"