feat: Add support for cache options in config files. (#2184)

# Add support for cache options in config files A new config section has been added, `cache`. As a rule, any options specified on the command line override options in the configuration files. ## `cspell.json` ```ts { cache?: CacheSettings; } ``` ## CacheSettings: ```ts { /** * Store the results of processed files in order to only operate on the changed ones. * @default false */ useCache?: boolean; // cspell:word cspellcache /** * Path to the cache location. Can be a file or a directory. * If none specified `.cspellcache` will be used. * Relative paths are relative to the config file in which */ cacheLocation?: FSPathResolvable; /** * Strategy to use for detecting changed files, default: metadata * @default 'metadata' */ cacheStrategy?: CacheStrategy; } ``` ## New CLI Option `--no-cache` - `--no-cache` ensures that the cache is not used even if `cspell.json` has caching enabled. ## Misc * Improve testing of cache options in application. * Add cli option `--no-cache`
streetsidesoftware · Jan 6, 2022 · 7256919 · 7256919
1 parent 0cf54cc
commit 7256919
Show file tree

Hide file tree

Showing 34 changed files with 531 additions and 116 deletions.
diff --git a/cspell.json b/cspell.json
@@ -1,6 +1,10 @@
 {
-    "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json",
+    "$schema": "./cspell.schema.json",
     "version": "0.2",
+    "cache": {
+        "cacheLocation": "./.cspell/.cspellcache",
+        "cacheStrategy": "content"
+    },
     "dictionaryDefinitions": [
         {
             "name": "workspace",
@@ -15,8 +19,8 @@
     ],
     "ignorePaths": [
         ".eslintignore",
-        ".gitattributes",
         ".git",
+        ".gitattributes",
         ".gitignore",
         ".pnp.{js,cjs}",
         ".prettierignore",
@@ -32,6 +36,7 @@
         "**/jest.config.js",
         "**/node_modules/**",
         "**/tsconfig.json",
+        "/.cspell",
         "cspell.json",
         "integration-tests/config/config.json",
         "integration-tests/config/repositories/**",
@@ -50,7 +55,5 @@
     "ignoreWords": [
         "commitcomment"
     ],
-    "words": [
-        "medicalterms"
-    ]
+    "words": []
 }
diff --git a/cspell.schema.json b/cspell.schema.json
@@ -2,6 +2,34 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "additionalProperties": false,
   "definitions": {
+    "CacheSettings": {
+      "additionalProperties": false,
+      "properties": {
+        "cacheLocation": {
+          "$ref": "#/definitions/FSPathResolvable",
+          "description": "Path to the cache location. Can be a file or a directory. If none specified `.cspellcache` will be used. Relative paths are relative to the config file in which"
+        },
+        "cacheStrategy": {
+          "$ref": "#/definitions/CacheStrategy",
+          "default": "metadata",
+          "description": "Strategy to use for detecting changed files, default: metadata"
+        },
+        "useCache": {
+          "default": false,
+          "description": "Store the results of processed files in order to only operate on the changed ones.",
+          "type": "boolean"
+        }
+      },
+      "type": "object"
+    },
+    "CacheStrategy": {
+      "description": "The Strategy to use to detect if a file has changed.\n- `metadata` - uses the file system timestamp and size to detect changes (fastest).\n- `content` - uses a hash of the file content to check file changes (slower - more accurate).",
+      "enum": [
+        "metadata",
+        "content"
+      ],
+      "type": "string"
+    },
     "CustomDictionaryPath": {
       "description": "A File System Path to a dictionary file.",
       "pattern": "^.*\\.txt$",
@@ -210,8 +238,12 @@
       ],
       "description": "Reference to a dictionary by name. One of:\n-  {@link  DictionaryRef } \n-  {@link  DictionaryNegRef }"
     },
+    "FSPathResolvable": {
+      "$ref": "#/definitions/FsPath",
+      "description": "A File System Path.\n\nSpecial Properties:\n- `${cwd}` prefix - will be replaced with the current working directory.\n- Relative paths are relative to the configuration file."
+    },
     "FsPath": {
-      "description": "A File System Path.",
+      "description": "A File System Path. Relative paths are relative to the configuration file.",
       "type": "string"
     },
     "Glob": {
@@ -742,6 +774,10 @@
       "description": "True to enable compound word checking.",
       "type": "boolean"
     },
+    "cache": {
+      "$ref": "#/definitions/CacheSettings",
+      "description": "Define cache settings."
+    },
     "caseSensitive": {
       "default": false,
       "description": "Determines if words must match case and accent rules.\n\n- `false` - Case is ignored and accents can be missing on the entire word.   Incorrect accents or partially missing accents will be marked as incorrect.\n- `true` - Case and accents are enforced.",
@@ -821,7 +857,7 @@
       "description": "Tells the spell checker to searching for `.gitignore` files when it reaches a matching root."
     },
     "globRoot": {
-      "$ref": "#/definitions/FsPath",
+      "$ref": "#/definitions/FSPathResolvable",
       "description": "The root to use for glob patterns found in this configuration. Default: location of the configuration file.   For compatibility reasons, config files with version 0.1, the glob root will   default to be `${cwd}`.\n\nUse `globRoot` to define a different location. `globRoot` can be relative to the location of this configuration file. Defining globRoot, does not impact imported configurations.\n\nSpecial Values:\n- `${cwd}` - will be replaced with the current working directory.\n- `.` - will be the location of the containing configuration file."
     },
     "id": {

diff --git a/packages/cspell-json-reporter/src/__snapshots__/index.test.ts.snap b/packages/cspell-json-reporter/src/__snapshots__/index.test.ts.snap
@@ -64,7 +64,8 @@ exports[`getReporter saves additional data 1`] = `
             \\"text.txt\\"
         ],
         \\"issues\\": 2,
-        \\"errors\\": 1
+        \\"errors\\": 1,
+        \\"cachedFiles\\": 0
     }
 }"
 `;
@@ -108,7 +109,8 @@ exports[`getReporter saves json to file 1`] = `
             \\"text.txt\\"
         ],
         \\"issues\\": 2,
-        \\"errors\\": 1
+        \\"errors\\": 1,
+        \\"cachedFiles\\": 0
     }
 }"
 `;

diff --git a/packages/cspell-json-reporter/src/index.test.ts b/packages/cspell-json-reporter/src/index.test.ts
@@ -78,5 +78,6 @@ async function runReporter(reporter: CSpellReporter): Promise<void> {
         filesWithIssues: new Set(['text.txt']),
         issues: 2,
         errors: 1,
+        cachedFiles: 0,
     });
 }
diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts
@@ -1,6 +1,5 @@
 import type { CSpellSettingsWithSourceTrace, CSpellUserSettings, ImportFileRef } from '@cspell/cspell-types';
 import * as path from 'path';
-import { mocked } from 'ts-jest/utils';
 import { URI } from 'vscode-uri';
 import { logError, logWarning } from '../util/logger';
 import {
@@ -26,7 +25,7 @@ import {
 } from './CSpellSettingsServer';
 import { getDefaultSettings, _defaultSettings } from './DefaultSettings';
 
-const { normalizeSettings, validateRawConfigVersion, validateRawConfigExports } = __testing__;
+const { normalizeCacheSettings, normalizeSettings, validateRawConfigExports, validateRawConfigVersion } = __testing__;
 
 const rootCspellLib = path.resolve(path.join(__dirname, '../..'));
 const root = path.resolve(rootCspellLib, '../..');
@@ -36,8 +35,8 @@ const testFixtures = path.join(rootCspellLib, '../../test-fixtures');
 
 jest.mock('../util/logger');
 
-const mockedLogError = mocked(logError);
-const mockedLogWarning = mocked(logWarning);
+const mockedLogError = jest.mocked(logError);
+const mockedLogWarning = jest.mocked(logWarning);
 
 describe('Validate CSpellSettingsServer', () => {
     test('tests mergeSettings with conflicting "name"', () => {
@@ -629,6 +628,20 @@ describe('Validate search/load config files', () => {
     });
 });
 
+describe('Validate Normalize Settings', () => {
+    test.each`
+        config                                           | expected
+        ${{}}                                            | ${{}}
+        ${{ cache: {} }}                                 | ${{ cache: {} }}
+        ${{ cache: { useCache: false } }}                | ${{ cache: { useCache: false } }}
+        ${{ cache: { useCache: undefined } }}            | ${{ cache: { useCache: undefined } }}
+        ${{ cache: { cacheLocation: '.cache' } }}        | ${{ cache: { cacheLocation: r(root, '.cache') } }}
+        ${{ cache: { cacheLocation: '${cwd}/.cache' } }} | ${{ cache: { cacheLocation: r(process.cwd(), '.cache') } }}
+    `('normalizeCacheSettings', ({ config, expected }) => {
+        expect(normalizeCacheSettings(config, root)).toEqual(expected);
+    });
+});
+
 describe('Validate Dependencies', () => {
     test.each`
         filename                         | relativeTo   | expected

diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts
@@ -163,6 +163,7 @@ function normalizeSettings(
     const normalizedOverrides = normalizeOverrides(settings, pathToSettingsFile);
     const normalizedReporters = normalizeReporters(settings, pathToSettingsFile);
     const normalizedGitignoreRoot = normalizeGitignoreRoot(settings, pathToSettingsFile);
+    const normalizedCacheSettings = normalizeCacheSettings(settings, pathToSettingsDir);
 
     const imports = typeof settings.import === 'string' ? [settings.import] : settings.import || [];
     const source: Source = settings.source || {
@@ -178,6 +179,7 @@ function normalizeSettings(
         ...normalizedOverrides,
         ...normalizedReporters,
         ...normalizedGitignoreRoot,
+        ...normalizedCacheSettings,
     };
     if (!imports.length) {
         return fileSettings;
@@ -713,6 +715,12 @@ function resolveGlobRoot(settings: CSpellSettings, pathToSettingsFile: string):
     return globRoot;
 }
 
+function resolveFilePath(filename: string, pathToSettingsFile: string): string {
+    const cwd = process.cwd();
+
+    return path.resolve(pathToSettingsFile, filename.replace('${cwd}', cwd));
+}
+
 function toGlobDef(g: undefined, root: string | undefined, source: string | undefined): undefined;
 function toGlobDef(g: Glob, root: string | undefined, source: string | undefined): GlobDef;
 function toGlobDef(g: Glob[], root: string | undefined, source: string | undefined): GlobDef[];
@@ -854,6 +862,17 @@ function normalizeSettingsGlobs(
     };
 }
 
+function normalizeCacheSettings(
+    settings: Pick<CSpellSettings, 'cache'>,
+    pathToSettingsDir: string
+): Pick<CSpellSettings, 'cache'> {
+    const { cache } = settings;
+    if (cache === undefined) return {};
+    const { cacheLocation } = cache;
+    if (cacheLocation === undefined) return { cache };
+    return { cache: { ...cache, cacheLocation: resolveFilePath(cacheLocation, pathToSettingsDir) } };
+}
+
 function validationMessage(msg: string, fileRef: ImportFileRef) {
     return msg + `\n  File: "${fileRef.filename}"`;
 }
@@ -907,7 +926,8 @@ function validateRawConfig(config: CSpellUserSettings, fileRef: ImportFileRef):
 }
 
 export const __testing__ = {
+    normalizeCacheSettings,
     normalizeSettings,
-    validateRawConfigVersion,
     validateRawConfigExports,
+    validateRawConfigVersion,
 };
diff --git a/packages/cspell-lib/src/Settings/GlobalSettings.test.ts b/packages/cspell-lib/src/Settings/GlobalSettings.test.ts
@@ -1,5 +1,4 @@
 import Configstore from 'configstore';
-import { mocked } from 'ts-jest/utils';
 import { getLogger } from '../util/logger';
 // eslint-disable-next-line jest/no-mocks-import
 import {
@@ -14,7 +13,7 @@ import { getGlobalConfigPath, getRawGlobalSettings, writeRawGlobalSettings } fro
 const logger = getLogger();
 const mockLog = jest.spyOn(logger, 'log').mockImplementation();
 const mockError = jest.spyOn(logger, 'error').mockImplementation();
-const mockConfigstore = mocked(Configstore, true);
+const mockConfigstore = jest.mocked(Configstore, true);
 
 describe('Validate GlobalSettings', () => {
     beforeEach(() => {

diff --git a/packages/cspell-lib/src/util/logger.test.ts b/packages/cspell-lib/src/util/logger.test.ts
@@ -1,4 +1,3 @@
-import { mocked } from 'ts-jest/utils';
 import { getLogger, log, logError, Logger, logWarning, setLogger } from './logger';
 
 const logger: Logger = {
@@ -7,7 +6,7 @@ const logger: Logger = {
     error: jest.fn(),
 };
 
-const mockLogger = mocked(logger);
+const mockLogger = jest.mocked(logger);
 
 describe('logger', () => {
     beforeEach(() => {

diff --git a/packages/cspell-types/cspell.schema.json b/packages/cspell-types/cspell.schema.json
@@ -2,6 +2,34 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "additionalProperties": false,
   "definitions": {
+    "CacheSettings": {
+      "additionalProperties": false,
+      "properties": {
+        "cacheLocation": {
+          "$ref": "#/definitions/FSPathResolvable",
+          "description": "Path to the cache location. Can be a file or a directory. If none specified `.cspellcache` will be used. Relative paths are relative to the config file in which"
+        },
+        "cacheStrategy": {
+          "$ref": "#/definitions/CacheStrategy",
+          "default": "metadata",
+          "description": "Strategy to use for detecting changed files, default: metadata"
+        },
+        "useCache": {
+          "default": false,
+          "description": "Store the results of processed files in order to only operate on the changed ones.",
+          "type": "boolean"
+        }
+      },
+      "type": "object"
+    },
+    "CacheStrategy": {
+      "description": "The Strategy to use to detect if a file has changed.\n- `metadata` - uses the file system timestamp and size to detect changes (fastest).\n- `content` - uses a hash of the file content to check file changes (slower - more accurate).",
+      "enum": [
+        "metadata",
+        "content"
+      ],
+      "type": "string"
+    },
     "CustomDictionaryPath": {
       "description": "A File System Path to a dictionary file.",
       "pattern": "^.*\\.txt$",
@@ -210,8 +238,12 @@
       ],
       "description": "Reference to a dictionary by name. One of:\n-  {@link  DictionaryRef } \n-  {@link  DictionaryNegRef }"
     },
+    "FSPathResolvable": {
+      "$ref": "#/definitions/FsPath",
+      "description": "A File System Path.\n\nSpecial Properties:\n- `${cwd}` prefix - will be replaced with the current working directory.\n- Relative paths are relative to the configuration file."
+    },
     "FsPath": {
-      "description": "A File System Path.",
+      "description": "A File System Path. Relative paths are relative to the configuration file.",
       "type": "string"
     },
     "Glob": {
@@ -742,6 +774,10 @@
       "description": "True to enable compound word checking.",
       "type": "boolean"
     },
+    "cache": {
+      "$ref": "#/definitions/CacheSettings",
+      "description": "Define cache settings."
+    },
     "caseSensitive": {
       "default": false,
       "description": "Determines if words must match case and accent rules.\n\n- `false` - Case is ignored and accents can be missing on the entire word.   Incorrect accents or partially missing accents will be marked as incorrect.\n- `true` - Case and accents are enforced.",
@@ -821,7 +857,7 @@
       "description": "Tells the spell checker to searching for `.gitignore` files when it reaches a matching root."
     },
     "globRoot": {
-      "$ref": "#/definitions/FsPath",
+      "$ref": "#/definitions/FSPathResolvable",
       "description": "The root to use for glob patterns found in this configuration. Default: location of the configuration file.   For compatibility reasons, config files with version 0.1, the glob root will   default to be `${cwd}`.\n\nUse `globRoot` to define a different location. `globRoot` can be relative to the location of this configuration file. Defining globRoot, does not impact imported configurations.\n\nSpecial Values:\n- `${cwd}` - will be replaced with the current working directory.\n- `.` - will be the location of the containing configuration file."
     },
     "id": {

diff --git a/packages/cspell-types/src/CSpellReporter.ts b/packages/cspell-types/src/CSpellReporter.ts
@@ -49,10 +49,16 @@ export interface ProgressFileComplete extends ProgressBase {
 export type ProgressEmitter = (p: ProgressItem | ProgressFileComplete) => void;
 
 export interface RunResult {
+    /** Number of files processed. */
     files: number;
+    /** Set of files where issues were found. */
     filesWithIssues: Set<string>;
+    /** Number of issues found. */
     issues: number;
+    /** Number of processing errors. */
     errors: number;
+    /** Number files that used results from the cache. */
+    cachedFiles: number;
 }
 
 export type ResultEmitter = (result: RunResult) => void | Promise<void>;