fix: allow html syntax in MDX v2 with format md (#8960)

* attempt to support html embeds in mdx with format md

* refactor mdx loader + support embedding html in commonmark thanks to rehype-raw

* extract processor code

* refactor processor code

* extract format + unit test

* try to refactor processor

* try to refactor processor

* adjust md page

* do not apply rehype-raw when format is mdx

* fix lint issue

Author: slorber

jest/deps.d.ts

@@ -13,10 +13,6 @@ declare module 'to-vfile' {
   export function read(path: string, encoding?: string): Promise<VFile>;
 }
 
-declare module 'remark-rehype';
-
-declare module 'rehype-stringify';
-
 declare module '@testing-utils/git' {
   const createTempRepo: typeof import('./utils/git').createTempRepo;
   export {createTempRepo};

packages/docusaurus-mdx-loader/package.json

@@ -32,6 +32,7 @@
     "image-size": "^1.0.2",
     "mdast-util-to-string": "^3.0.0",
     "mdast-util-mdx": "^2.0.0",
+    "rehype-raw": "^6.1.1",
     "remark-comment": "^1.0.0",
     "remark-gfm": "^3.0.1",
     "remark-directive": "^2.0.1",

packages/docusaurus-mdx-loader/src/__tests__/format.test.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {getFormat} from '../format';
+
+describe('getFormat', () => {
+  it('uses frontMatter format over anything else', () => {
+    expect(getFormat({frontMatterFormat: 'md', filePath: 'xyz.md'})).toBe('md');
+    expect(getFormat({frontMatterFormat: 'md', filePath: 'xyz.mdx'})).toBe(
+      'md',
+    );
+    expect(getFormat({frontMatterFormat: 'mdx', filePath: 'xyz.md'})).toBe(
+      'mdx',
+    );
+    expect(getFormat({frontMatterFormat: 'mdx', filePath: 'xyz.mdx'})).toBe(
+      'mdx',
+    );
+  });
+
+  it('detects appropriate format from file extension', () => {
+    expect(getFormat({frontMatterFormat: 'detect', filePath: 'xyz.md'})).toBe(
+      'md',
+    );
+    expect(
+      getFormat({frontMatterFormat: 'detect', filePath: 'xyz.markdown'}),
+    ).toBe('md');
+
+    expect(
+      getFormat({frontMatterFormat: 'detect', filePath: 'folder/xyz.md'}),
+    ).toBe('md');
+    expect(
+      getFormat({frontMatterFormat: 'detect', filePath: 'folder/xyz.markdown'}),
+    ).toBe('md');
+    expect(getFormat({frontMatterFormat: 'detect', filePath: 'xyz.mdx'})).toBe(
+      'mdx',
+    );
+    expect(
+      getFormat({frontMatterFormat: 'detect', filePath: 'folder/xyz.mdx'}),
+    ).toBe('mdx');
+
+    expect(
+      getFormat({frontMatterFormat: 'detect', filePath: 'xyz.unknown'}),
+    ).toBe('mdx');
+    expect(
+      getFormat({frontMatterFormat: 'detect', filePath: 'folder/xyz.unknown'}),
+    ).toBe('mdx');
+  });
+});

packages/docusaurus-mdx-loader/src/__tests__/processor.test.ts

@@ -0,0 +1,33 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// import {createProcessor} from '../processor';
+// import type {Options} from '../loader';
+
+/*
+async function testProcess({
+  format,
+  options,
+}: {
+  format: 'md' | 'mdx';
+  options: Options;
+}) {
+  return async (content: string) => {
+    const processor = await createProcessor({format, options});
+    return processor.process(content);
+  };
+}
+ */
+
+describe('md processor', () => {
+  it('parses simple commonmark', async () => {
+    // TODO no tests for now, wait until ESM support
+    // Jest does not support well ESM modules
+    // It would require to vendor too much Unified modules as CJS
+    // See https://mdxjs.com/docs/troubleshooting-mdx/#esm
+  });
+});

packages/docusaurus-mdx-loader/src/format.ts

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import path from 'path';
+import type {MDXFrontMatter} from './frontMatter';
+
+// Copied from https://mdxjs.com/packages/mdx/#optionsmdextensions
+// Although we are likely to only use .md / .mdx anyway...
+const mdFormatExtensions = [
+  '.md',
+  '.markdown',
+  '.mdown',
+  '.mkdn',
+  '.mkd',
+  '.mdwn',
+  '.mkdown',
+  '.ron',
+];
+
+function isMDFormat(filepath: string) {
+  return mdFormatExtensions.includes(path.extname(filepath));
+}
+
+export function getFormat({
+  filePath,
+  frontMatterFormat,
+}: {
+  filePath: string;
+  frontMatterFormat: MDXFrontMatter['format'];
+}): 'md' | 'mdx' {
+  if (frontMatterFormat !== 'detect') {
+    return frontMatterFormat;
+  }
+  // Bias toward mdx if unknown extension
+  return isMDFormat(filePath) ? 'md' : 'mdx';
+}

packages/docusaurus-mdx-loader/src/index.ts

@@ -30,4 +30,6 @@ export type LoadedMDXContent<FrontMatter, Metadata, Assets = undefined> = {
   readonly assets: Assets;
   (): JSX.Element;
 };
-export type {Options, MDXPlugin, MDXOptions} from './loader';
+
+export type {Options, MDXPlugin} from './loader';
+export type {MDXOptions} from './processor';