feat: register custom formats take 2 (decaporg#6205)

* feat: register custom formats

* fix: register custom formats validation

* fix: change 'format' field validation to string instead of enum

Format will have the same behaviour as the widget property

* test: custom formats and register function

* docs: explain usage and note manual initialization requirement

* fix: remove unused imports

* use default extension

* remove manual init note

* PR comments

* fix: prettier

* revert unnecessary changes?

* chore: more revert?

* chore: newline

* chore: update import

---------

Co-authored-by: Jean <[email protected]>
Co-authored-by: Misha Kaletsky <[email protected]>
Co-authored-by: Erez Rokah <[email protected]>

Author: 3 people

packages/decap-cms-core/index.d.ts

@@ -36,15 +36,7 @@ declare module 'decap-cms-core' {
     value: any;
   }
 
-  export type CmsCollectionFormatType =
-    | 'yml'
-    | 'yaml'
-    | 'toml'
-    | 'json'
-    | 'frontmatter'
-    | 'yaml-frontmatter'
-    | 'toml-frontmatter'
-    | 'json-frontmatter';
+  export type CmsCollectionFormatType = string;
 
   export type CmsAuthScope = 'repo' | 'public_repo';
 
@@ -501,6 +493,11 @@ declare module 'decap-cms-core' {
 
   export type CmsLocalePhrases = any; // TODO: type properly
 
+  export type Formatter = {
+    fromFile(content: string): unknown;
+    toFile(data: object, sortedKeys?: string[], comments?: Record<string, string>): string;
+  };
+
   export interface CmsRegistry {
     backends: {
       [name: string]: CmsRegistryBackend;
@@ -520,6 +517,9 @@ declare module 'decap-cms-core' {
     locales: {
       [name: string]: CmsLocalePhrases;
     };
+    formats: {
+      [name: string]: Formatter;
+    };
   }
 
   type GetAssetFunction = (asset: string) => {
@@ -579,6 +579,7 @@ declare module 'decap-cms-core' {
       serializer: CmsWidgetValueSerializer,
     ) => void;
     resolveWidget: (name: string) => CmsWidget | undefined;
+    registerCustomFormat: (name: string, extension: string, formatter: Formatter) => void;
   }
 
   export const DecapCmsCore: CMS;

packages/decap-cms-core/src/constants/configSchema.js

@@ -8,7 +8,7 @@ import {
 import ajvErrors from 'ajv-errors';
 import uuid from 'uuid/v4';
 
-import { formatExtensions, frontmatterFormats, extensionFormatters } from '../formats/formats';
+import { frontmatterFormats, extensionFormatters } from '../formats/formats';
 import { getWidgets } from '../lib/registry';
 import { I18N_STRUCTURE, I18N_FIELD } from '../lib/i18n';
 
@@ -231,7 +231,7 @@ function getConfigSchema() {
                 preview: { type: 'boolean' },
               },
             },
-            format: { type: 'string', enum: Object.keys(formatExtensions) },
+            format: { type: 'string' },
             extension: { type: 'string' },
             frontmatter_delimiter: {
               type: ['string', 'array'],

packages/decap-cms-core/src/formats/__tests__/formats.spec.js

@@ -0,0 +1,87 @@
+import { Map } from 'immutable';
+
+import { extensionFormatters, resolveFormat } from '../formats';
+import { registerCustomFormat } from '../../lib/registry';
+
+describe('custom formats', () => {
+  const testEntry = {
+    collection: 'testCollection',
+    data: { x: 1 },
+    isModification: false,
+    label: 'testLabel',
+    mediaFiles: [],
+    meta: {},
+    newRecord: true,
+    partial: false,
+    path: 'testPath1',
+    raw: 'testRaw',
+    slug: 'testSlug',
+    author: 'testAuthor',
+    updatedOn: 'testUpdatedOn',
+  };
+  it('resolves builtint formats', () => {
+    const collection = Map({
+      name: 'posts',
+    });
+    expect(resolveFormat(collection, { ...testEntry, path: 'test.yml' })).toEqual(
+      extensionFormatters.yml,
+    );
+    expect(resolveFormat(collection, { ...testEntry, path: 'test.yaml' })).toEqual(
+      extensionFormatters.yml,
+    );
+    expect(resolveFormat(collection, { ...testEntry, path: 'test.toml' })).toEqual(
+      extensionFormatters.toml,
+    );
+    expect(resolveFormat(collection, { ...testEntry, path: 'test.json' })).toEqual(
+      extensionFormatters.json,
+    );
+    expect(resolveFormat(collection, { ...testEntry, path: 'test.md' })).toEqual(
+      extensionFormatters.md,
+    );
+    expect(resolveFormat(collection, { ...testEntry, path: 'test.markdown' })).toEqual(
+      extensionFormatters.markdown,
+    );
+    expect(resolveFormat(collection, { ...testEntry, path: 'test.html' })).toEqual(
+      extensionFormatters.html,
+    );
+  });
+
+  it('resolves custom format', () => {
+    registerCustomFormat('txt-querystring', 'txt', {
+      fromFile: file => Object.fromEntries(new URLSearchParams(file)),
+      toFile: value => new URLSearchParams(value).toString(),
+    });
+
+    const collection = Map({
+      name: 'posts',
+      format: 'txt-querystring',
+    });
+
+    const formatter = resolveFormat(collection, { ...testEntry, path: 'test.txt' });
+
+    expect(formatter.toFile({ foo: 'bar' })).toEqual('foo=bar');
+    expect(formatter.fromFile('foo=bar')).toEqual({ foo: 'bar' });
+  });
+
+  it('can override existing formatters', () => {
+    // simplified version of a more realistic use case: using a different yaml library like js-yaml
+    // to make netlify-cms play nice with other tools that edit content and spit out yaml
+    registerCustomFormat('bad-yaml', 'yml', {
+      fromFile: file => Object.fromEntries(file.split('\n').map(line => line.split(': '))),
+      toFile: value =>
+        Object.entries(value)
+          .map(([k, v]) => `${k}: ${v}`)
+          .join('\n'),
+    });
+
+    const collection = Map({
+      name: 'posts',
+      format: 'bad-yaml',
+    });
+
+    const formatter = resolveFormat(collection, { ...testEntry, path: 'test.txt' });
+
+    expect(formatter.toFile({ a: 'b', c: 'd' })).toEqual('a: b\nc: d');
+    expect(formatter.fromFile('a: b\nc: d')).toEqual({ a: 'b', c: 'd' });
+  });
+});

packages/decap-cms-core/src/formats/formats.ts

@@ -5,10 +5,12 @@ import yamlFormatter from './yaml';
 import tomlFormatter from './toml';
 import jsonFormatter from './json';
 import { FrontmatterInfer, frontmatterJSON, frontmatterTOML, frontmatterYAML } from './frontmatter';
+import { getCustomFormatsExtensions, getCustomFormatsFormatters } from '../lib/registry';
 
 import type { Delimiter } from './frontmatter';
 import type { Collection, EntryObject, Format } from '../types/redux';
 import type { EntryValue } from '../valueObjects/Entry';
+import type { Formatter } from 'decap-cms-core';
 
 export const frontmatterFormats = ['yaml-frontmatter', 'toml-frontmatter', 'json-frontmatter'];
 
@@ -23,6 +25,10 @@ export const formatExtensions = {
   'yaml-frontmatter': 'md',
 };
 
+export function getFormatExtensions() {
+  return { ...formatExtensions, ...getCustomFormatsExtensions() };
+}
+
 export const extensionFormatters = {
   yml: yamlFormatter,
   yaml: yamlFormatter,
@@ -33,8 +39,8 @@ export const extensionFormatters = {
   html: FrontmatterInfer,
 };
 
-function formatByName(name: Format, customDelimiter?: Delimiter) {
-  return {
+function formatByName(name: Format, customDelimiter?: Delimiter): Formatter {
+  const formatters: Record<string, Formatter> = {
     yml: yamlFormatter,
     yaml: yamlFormatter,
     toml: tomlFormatter,
@@ -43,7 +49,12 @@ function formatByName(name: Format, customDelimiter?: Delimiter) {
     'json-frontmatter': frontmatterJSON(customDelimiter),
     'toml-frontmatter': frontmatterTOML(customDelimiter),
     'yaml-frontmatter': frontmatterYAML(customDelimiter),
-  }[name];
+    ...getCustomFormatsFormatters(),
+  };
+  if (name in formatters) {
+    return formatters[name];
+  }
+  throw new Error(`No formatter available with name: ${name}`);
 }
 
 function frontmatterDelimiterIsList(

packages/decap-cms-core/src/lib/__tests__/registry.spec.js

@@ -46,6 +46,21 @@ describe('registry', () => {
     });
   });
 
+  describe('registerCustomFormat', () => {
+    it('can register a custom format', () => {
+      const { getCustomFormats, registerCustomFormat } = require('../registry');
+
+      expect(Object.keys(getCustomFormats())).not.toContain('querystring');
+
+      registerCustomFormat('querystring', 'qs', {
+        fromFile: content => Object.fromEntries(new URLSearchParams(content)),
+        toFile: obj => new URLSearchParams(obj).toString(),
+      });
+
+      expect(Object.keys(getCustomFormats())).toContain('querystring');
+    });
+  });
+
   describe('eventHandlers', () => {
     const events = [
       'prePublish',

packages/decap-cms-core/src/lib/registry.js

@@ -31,6 +31,7 @@ const registry = {
   mediaLibraries: [],
   locales: {},
   eventHandlers,
+  formats: {},
 };
 
 export default {
@@ -58,6 +59,10 @@ export default {
   removeEventListener,
   getEventListeners,
   invokeEvent,
+  registerCustomFormat,
+  getCustomFormats,
+  getCustomFormatsExtensions,
+  getCustomFormatsFormatters,
 };
 
 /**
@@ -280,3 +285,28 @@ export function registerLocale(locale, phrases) {
 export function getLocale(locale) {
   return registry.locales[locale];
 }
+
+export function registerCustomFormat(name, extension, formatter) {
+  registry.formats[name] = { extension, formatter };
+}
+
+export function getCustomFormats() {
+  return registry.formats;
+}
+
+export function getCustomFormatsExtensions() {
+  return Object.entries(registry.formats).reduce(function (acc, [name, { extension }]) {
+    return { ...acc, [name]: extension };
+  }, {});
+}
+
+/** @type {() => Record<string, unknown>} */
+export function getCustomFormatsFormatters() {
+  return Object.entries(registry.formats).reduce(function (acc, [name, { formatter }]) {
+    return { ...acc, [name]: formatter };
+  }, {});
+}
+
+export function getFormatter(name) {
+  return registry.formats[name]?.formatter;
+}

packages/decap-cms-core/src/reducers/collections.ts

@@ -7,7 +7,7 @@ import { CONFIG_SUCCESS } from '../actions/config';
 import { FILES, FOLDER } from '../constants/collectionTypes';
 import { COMMIT_DATE, COMMIT_AUTHOR } from '../constants/commitProps';
 import { INFERABLE_FIELDS, IDENTIFIER_FIELDS, SORTABLE_FIELDS } from '../constants/fieldInference';
-import { formatExtensions } from '../formats/formats';
+import { getFormatExtensions } from '../formats/formats';
 import { selectMediaFolder } from './entries';
 import { summaryFormatter } from '../lib/formatters';
 
@@ -46,10 +46,14 @@ function collections(state = defaultState, action: ConfigAction) {
 const selectors = {
   [FOLDER]: {
     entryExtension(collection: Collection) {
-      return (
+      const ext =
         collection.get('extension') ||
-        get(formatExtensions, collection.get('format') || 'frontmatter')
-      ).replace(/^\./, '');
+        get(getFormatExtensions(), collection.get('format') || 'frontmatter');
+      if (!ext) {
+        throw new Error(`No extension found for format ${collection.get('format')}`);
+      }
+
+      return ext.replace(/^\./, '');
     },
     fields(collection: Collection) {
       return collection.get('fields');

packages/decap-cms-core/src/types/redux.ts

@@ -599,7 +599,7 @@ type i18n = StaticallyTypedRecord<{
   default_locale: string;
 }>;
 
-export type Format = keyof typeof formatExtensions;
+export type Format = keyof typeof formatExtensions | string;
 
 type CollectionObject = {
   name: string;

website/content/docs/beta-features.md

@@ -684,3 +684,30 @@ CMS.registerRemarkPlugin({ settings: { bullet: '-' } });
 ```
 
 Note that `netlify-widget-markdown` currently uses `remark@10`, so you should check a plugin's compatibility first.
+
+## Custom formatters
+
+To manage content with other file formats than the built in ones, you can register a custom formatter:
+
+```js
+const JSON5 = require('json5');
+
+CMS.registerCustomFormat('json5', 'json5', {
+  fromFile: text => JSON5.parse(text),
+  toFile: value => JSON5.stringify(value, null, 2),
+});
+```
+
+Then include `format: json5` in your collection configuration. See the [Collection docs](https://www.netlifycms.org/docs/configuration-options/#collections) for more details.
+
+You can also override the in-built formatters. For example, to change the YAML serialization method from [`yaml`](https://npmjs.com/package/yaml) to [`js-yaml`](https://npmjs.com/package/js-yaml):
+
+```js
+const jsYaml = require('js-yaml');
+
+CMS.registerCustomFormat('yml', 'yml', {
+  fromFile: text => jsYaml.load(text),
+  toFile: value => jsYaml.dump(value),
+});
+```
+