Add new image processing benchmark site (#16701)

benchmarks/image-processing/.gitignore

@@ -0,0 +1,69 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# dotenv environment variables file
+.env
+
+# gatsby files
+.cache/
+public
+
+# Mac files
+.DS_Store
+
+# Yarn
+yarn-error.log
+.pnp/
+.pnp.js
+# Yarn Integrity file
+.yarn-integrity

benchmarks/image-processing/.prettierignore

@@ -0,0 +1,4 @@
+.cache
+package.json
+package-lock.json
+public

benchmarks/image-processing/.prettierrc

@@ -0,0 +1,7 @@
+{
+  "endOfLine": "lf",
+  "semi": false,
+  "singleQuote": false,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

benchmarks/image-processing/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

benchmarks/image-processing/README.md

@@ -0,0 +1,17 @@
+# Image Processing benchmark
+
+Benchmarks Gatsby's image processing capabilities
+by downloading and processing images.
+
+Defaults to building a site with 100 image pages. Set the `NUM_IMAGES` environment variable to change that e.g. `NUM_IMAGES=1000 MAX_NUM_ROWS=100 gatsby build`
+
+The max number of images you can process is 1300.
+
+# Running the benchmark
+
+First, install node modules required by package.json. This is needed only one time. Then run the build
+
+```bash
+npm install
+npm run build
+```

benchmarks/image-processing/gatsby-browser.js

@@ -0,0 +1,7 @@
+/**
+ * Implement Gatsby's Browser APIs in this file.
+ *
+ * See: https://www.gatsbyjs.org/docs/browser-apis/
+ */
+
+// You can delete this file if you're not using it

benchmarks/image-processing/gatsby-config.js

@@ -0,0 +1,35 @@
+module.exports = {
+  siteMetadata: {
+    title: `Gatsby Default Starter`,
+    description: `Kick off your next, great Gatsby project with this default starter. This barebones starter ships with the main Gatsby configuration files you might need.`,
+    author: `@gatsbyjs`,
+  },
+  plugins: [
+    `gatsby-source-remote-images`,
+    `gatsby-plugin-react-helmet`,
+    {
+      resolve: `gatsby-source-filesystem`,
+      options: {
+        name: `images`,
+        path: `${__dirname}/src/images`,
+      },
+    },
+    `gatsby-transformer-sharp`,
+    `gatsby-plugin-sharp`,
+    {
+      resolve: `gatsby-plugin-manifest`,
+      options: {
+        name: `gatsby-starter-default`,
+        short_name: `starter`,
+        start_url: `/`,
+        background_color: `#663399`,
+        theme_color: `#663399`,
+        display: `minimal-ui`,
+        icon: `src/images/gatsby-icon.png`, // This path is relative to the root of the site.
+      },
+    },
+    // this (optional) plugin enables Progressive Web App + Offline functionality
+    // To learn more, visit: https://gatsby.dev/offline
+    // `gatsby-plugin-offline`,
+  ],
+}

benchmarks/image-processing/gatsby-node.js

@@ -0,0 +1,26 @@
+const path = require(`path`)
+
+exports.createPages = ({ actions, graphql }) => {
+  const { createPage } = actions
+
+  return graphql(`
+    query MyQuery {
+      allRemoteImage {
+        totalCount
+        nodes {
+          id
+        }
+      }
+    }
+  `).then(results => {
+    results.data.allRemoteImage.nodes.map(n => {
+      createPage({
+        path: `/${n.id}/`,
+        component: path.resolve(`./src/templates/image.js`),
+        context: {
+          id: n.id,
+        },
+      })
+    })
+  })
+}

benchmarks/image-processing/gatsby-ssr.js

@@ -0,0 +1,7 @@
+/**
+ * Implement Gatsby's SSR (Server Side Rendering) APIs in this file.
+ *
+ * See: https://www.gatsbyjs.org/docs/ssr-apis/
+ */
+
+// You can delete this file if you're not using it

benchmarks/image-processing/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "gatsby-benchmark-image-processing",
+  "private": true,
+  "description": "A benchmark site to test how fast Gatsby can process images",
+  "version": "0.1.0",
+  "author": "Kyle Mathews <mathews.kyle@gmail.com>",
+  "dependencies": {
+    "async-sema": "^3.0.0",
+    "gatsby": "^2.13.67",
+    "gatsby-image": "^2.2.8",
+    "gatsby-plugin-manifest": "^2.2.5",
+    "gatsby-plugin-offline": "^2.2.6",
+    "gatsby-plugin-react-helmet": "^3.1.3",
+    "gatsby-plugin-sharp": "^2.2.12",
+    "gatsby-source-filesystem": "^2.1.9",
+    "gatsby-transformer-sharp": "^2.2.6",
+    "lodash": "^4.17.15",
+    "node-fetch": "^2.6.0",
+    "prop-types": "^15.7.2",
+    "react": "^16.9.0",
+    "react-dom": "^16.9.0",
+    "react-helmet": "^5.2.1",
+    "unsplash-js": "^5.0.0"
+  },
+  "devDependencies": {
+    "prettier": "^1.18.2"
+  },
+  "keywords": [
+    "gatsby"
+  ],
+  "license": "MIT",
+  "scripts": {
+    "build": "gatsby build",
+    "develop": "gatsby develop",
+    "format": "prettier --write \"**/*.{js,jsx,json,md}\"",
+    "start": "npm run develop",
+    "serve": "gatsby serve",
+    "test": "echo \"Write tests! -> https://gatsby.dev/unit-testing\""
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gatsbyjs/gatsby-starter-default"
+  },
+  "bugs": {
+    "url": "https://github.com/gatsbyjs/gatsby/issues"
+  }
+}

benchmarks/image-processing/plugins/gatsby-source-remote-images/fetch-image-urls.js

@@ -0,0 +1,41 @@
+const { Sema } = require(`async-sema`)
+const fetch = require(`node-fetch`)
+const _ = require(`lodash`)
+const fs = require(`fs`)
+const path = require(`path`)
+
+const baseUrl = `http://www.splashbase.co/api/v1/images/`
+
+const s = new Sema(
+  10, // Allow 10 concurrent async calls
+  {
+    capacity: 100, // Prealloc space for 100 tokens
+  }
+)
+
+async function fetchData(x) {
+  await s.acquire()
+  try {
+    return await fetch(`${baseUrl}${x}`).then(res => res.json())
+  } finally {
+    s.release()
+  }
+}
+
+const main = async () => {
+  let data = await Promise.all(_.range(2000).map(fetchData))
+  data = data.filter(d => d.url).map(d => d.url)
+  data = data.map(d => {
+    return {
+      url: d,
+      parsed: path.parse(d),
+    }
+  })
+  data = data.filter(d => d.parsed.ext.toLowerCase().slice(0, 4) === `.jpg`)
+  data = data.map(d => d.url)
+  // console.log(data)
+  console.log(`total: ${data.length}`)
+  fs.writeFileSync(`./urls.json`, JSON.stringify(data, null, 4))
+}
+
+main()

benchmarks/image-processing/plugins/gatsby-source-remote-images/gatsby-node.js

@@ -0,0 +1,41 @@
+const fs = require(`fs`)
+const { createRemoteFileNode } = require(`gatsby-source-filesystem`)
+
+const NUM_IMAGES = parseInt(process.env.NUM_IMAGES || 100, 10)
+
+const urls = JSON.parse(
+  fs.readFileSync(`${__dirname}/urls.json`, `utf-8`)
+).slice(0, NUM_IMAGES)
+
+exports.sourceNodes = ({ actions, createNodeId, store, cache }) =>
+  Promise.all(
+    urls.map(async url => {
+      let fileNode
+      const nodeId = createNodeId(url)
+      try {
+        fileNode = await createRemoteFileNode({
+          url: url,
+          parentNodeId: nodeId,
+          store,
+          cache,
+          createNode: actions.createNode,
+          createNodeId,
+        })
+      } catch (e) {
+        console.log(e)
+        // Ignore
+      }
+
+      const node = {
+        id: nodeId,
+        remoteUrl: url,
+        internal: { type: `RemoteImage`, contentDigest: url },
+      }
+
+      if (fileNode) {
+        node.file___NODE = fileNode.id
+      }
+
+      actions.createNode(node)
+    })
+  )

benchmarks/image-processing/plugins/gatsby-source-remote-images/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "gatsby-source-remote-images",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC"
+}