feat: ensure readme is in correct format (#997)

Uses modules from the `mdast`/`remark` ecosystem to ensure our readmes
are in a predictable format while running `aegir check-project`.

Ensure the following are present and consistent:

- Header (taken from `package.json` name field)
- CI link (assumes unified CI)
- Strap line (taken from `package.json` description field)
- Table of contents
- Installation instructions
- License
- Contributing

Existing header/strap line/etc is overwritten, any other existing content is added after the installation instructions.

Also formats GH tables nicely and removes all instances of that weird whitespace character you get when you hold down `option` and press `space` which messes up the headers in some of our READMEs.

package.json

@@ -240,7 +240,20 @@
     "kleur": "^4.1.4",
     "lilconfig": "^2.0.5",
     "listr": "~0.14.2",
+    "mdast-util-from-markdown": "^1.2.0",
+    "mdast-util-gfm": "^2.0.1",
+    "mdast-util-gfm-footnote": "^1.0.1",
+    "mdast-util-gfm-strikethrough": "^1.0.1",
+    "mdast-util-gfm-table": "^1.0.4",
+    "mdast-util-gfm-task-list-item": "^1.0.1",
+    "mdast-util-to-markdown": "^1.3.0",
+    "mdast-util-toc": "^6.1.0",
     "merge-options": "^3.0.4",
+    "micromark-extension-gfm": "^2.0.1",
+    "micromark-extension-gfm-footnote": "^1.0.4",
+    "micromark-extension-gfm-strikethrough": "^1.0.4",
+    "micromark-extension-gfm-table": "^1.0.5",
+    "micromark-extension-gfm-task-list-item": "^1.0.3",
     "mocha": "^9.0.2",
     "npm-package-json-lint": "^6.3.0",
     "nyc": "^15.1.0",

src/check-project/check-readme.js

@@ -0,0 +1,219 @@
+
+/* eslint-disable no-console */
+
+import fs from 'fs'
+import path from 'path'
+import {
+  ensureFileHasContents
+} from './utils.js'
+import { fromMarkdown } from 'mdast-util-from-markdown'
+import { toMarkdown } from 'mdast-util-to-markdown'
+import { toc as makeToc } from 'mdast-util-toc'
+import { gfm } from 'micromark-extension-gfm'
+import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm'
+import { gfmTable } from 'micromark-extension-gfm-table'
+import { gfmTableFromMarkdown, gfmTableToMarkdown } from 'mdast-util-gfm-table'
+import { gfmFootnote } from 'micromark-extension-gfm-footnote'
+import { gfmFootnoteFromMarkdown, gfmFootnoteToMarkdown } from 'mdast-util-gfm-footnote'
+import { gfmStrikethrough } from 'micromark-extension-gfm-strikethrough'
+import { gfmStrikethroughFromMarkdown, gfmStrikethroughToMarkdown } from 'mdast-util-gfm-strikethrough'
+import { gfmTaskListItem } from 'micromark-extension-gfm-task-list-item'
+import { gfmTaskListItemFromMarkdown, gfmTaskListItemToMarkdown } from 'mdast-util-gfm-task-list-item'
+
+/**
+ * @param {*} pkg
+ * @param {string} repoUrl
+ * @param {string} defaultBranch
+ */
+const HEADER = (pkg, repoUrl, defaultBranch) => {
+  return `
+# ${pkg.name} <!-- omit in toc -->
+
+[![test & maybe release](${repoUrl}/actions/workflows/js-test-and-release.yml/badge.svg?branch=${defaultBranch})](${repoUrl}/actions/workflows/js-test-and-release.yml)
+
+> ${pkg.description}
+
+## Table of contents <!-- omit in toc -->
+`
+}
+
+/**
+ * @param {*} pkg
+ */
+const INSTALL = (pkg) => {
+  return `
+## Install
+
+\`\`\`console
+$ npm i ${pkg.name}
+\`\`\`
+  `
+}
+
+const LICENSE = `
+## License
+
+Licensed under either of
+
+ * Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / http://www.apache.org/licenses/LICENSE-2.0)
+ * MIT ([LICENSE-MIT](LICENSE-MIT) / http://opensource.org/licenses/MIT)
+
+## Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions.
+`
+
+/**
+ * @param {string} md
+ */
+function parseMarkdown (md) {
+  return fromMarkdown(md, {
+    extensions: [
+      gfm(),
+      gfmTable,
+      gfmFootnote(),
+      gfmStrikethrough(),
+      gfmTaskListItem
+    ],
+    mdastExtensions: [
+      gfmFromMarkdown(),
+      gfmTableFromMarkdown,
+      gfmFootnoteFromMarkdown(),
+      gfmStrikethroughFromMarkdown,
+      gfmTaskListItemFromMarkdown
+    ]
+  })
+}
+
+/**
+ *
+ * @param {import('mdast').Root | import('mdast').Content} tree
+ */
+function writeMarkdown (tree) {
+  return toMarkdown(tree, {
+    extensions: [
+      gfmToMarkdown(),
+      gfmTableToMarkdown(),
+      gfmFootnoteToMarkdown(),
+      gfmStrikethroughToMarkdown,
+      gfmTaskListItemToMarkdown
+    ],
+    bullet: '-',
+    listItemIndent: 'one'
+  })
+}
+
+/**
+ * @param {string} projectDir
+ * @param {string} repoUrl
+ * @param {string} defaultBranch
+ */
+export async function checkReadme (projectDir, repoUrl, defaultBranch) {
+  console.info('Check README files')
+
+  const pkg = JSON.parse(fs.readFileSync(path.join(projectDir, 'package.json'), {
+    encoding: 'utf-8'
+  }))
+
+  const readmePath = path.join(projectDir, 'README.md')
+  let readmeContents = ''
+
+  if (fs.existsSync(readmePath)) {
+    readmeContents = fs.readFileSync(path.join(projectDir, 'README.md'), {
+      encoding: 'utf-8'
+    })
+  }
+
+  // replace the magic OPTION+SPACE character that messes up headers
+  readmeContents = readmeContents.replaceAll(' ', ' ')
+
+  // parse the project's readme file
+  const file = parseMarkdown(readmeContents)
+
+  // create basic readme with heading, CI link, etc
+  const readme = parseMarkdown(HEADER(pkg, repoUrl, defaultBranch))
+
+  // remove existing header, CI link, etc
+  /** @type {import('mdast').Root} */
+  const parsedReadme = {
+    type: 'root',
+    children: []
+  }
+
+  let tocIndex = -1
+  let installIndex = -1
+  let licenseFound = false
+
+  file.children.forEach((child, index) => {
+    const rendered = writeMarkdown(child).toLowerCase()
+
+    if (child.type === 'heading' && index === 0) {
+      // skip heading
+      return
+    }
+
+    if (child.type === 'paragraph' && index === 1) {
+      // skip badges
+      return
+    }
+
+    if (child.type === 'blockquote' && tocIndex === -1 && installIndex === -1) {
+      // skip project overview
+      return
+    }
+
+    if (rendered.includes('## table of')) {
+      // skip toc header
+      tocIndex = index
+      return
+    }
+
+    if (tocIndex !== -1 && index === tocIndex + 1) {
+      // skip toc header
+      return
+    }
+
+    if (child.type === 'heading' && rendered.includes('install')) {
+      // skip install
+      installIndex = index
+      return
+    }
+
+    if (installIndex !== -1 && index === installIndex + 1) {
+      // skip install
+      return
+    }
+
+    if ((child.type === 'heading' && rendered.includes('license')) || licenseFound) {
+      licenseFound = true
+      return
+    }
+
+    parsedReadme.children.push(child)
+  })
+
+  const installation = parseMarkdown(INSTALL(pkg))
+  const license = parseMarkdown(LICENSE)
+
+  parsedReadme.children = [
+    ...installation.children,
+    ...parsedReadme.children,
+    ...license.children
+  ]
+
+  const toc = makeToc(parsedReadme, {
+    tight: true
+  })
+
+  if (toc.map == null) {
+    throw new Error('Could not create TOC for README.md')
+  }
+
+  readme.children = [
+    ...readme.children,
+    toc.map,
+    ...parsedReadme.children
+  ]
+
+  await ensureFileHasContents(projectDir, 'README.md', writeMarkdown(readme))
+}

src/check-project/index.js

@@ -13,6 +13,7 @@ import { typedCJSManifest } from './manifests/typed-cjs.js'
 import { checkLicenseFiles } from './check-licence-files.js'
 import { checkBuildFiles } from './check-build-files.js'
 import { checkMonorepoFiles } from './check-monorepo-files.js'
+import { checkReadme } from './check-readme.js'
 import {
   sortManifest,
   ensureFileHasContents
@@ -394,6 +395,7 @@ async function processModule (projectDir, manifest, branchName, repoUrl, homePag
 
   await ensureFileHasContents(projectDir, 'package.json', JSON.stringify(proposedManifest, null, 2))
   await checkLicenseFiles(projectDir)
+  await checkReadme(projectDir, repoUrl, branchName)
 }
 
 export default new Listr([