From b9f73356e59b4fc1cd1a4cc64692886f6178f4a8 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 2 May 2019 19:21:05 +0200 Subject: [PATCH 01/37] wip #370 --- packages/wotan/src/dependency-resolver.ts | 227 ++++++++++++++++++++++ packages/wotan/src/linter.ts | 7 +- packages/wotan/src/program-state.ts | 206 ++++++++++++++++++++ packages/wotan/src/runner.ts | 49 +++-- packages/wotan/src/state-persistence.ts | 45 +++++ packages/wotan/src/utils.ts | 16 ++ 6 files changed, 533 insertions(+), 17 deletions(-) create mode 100644 packages/wotan/src/dependency-resolver.ts create mode 100644 packages/wotan/src/program-state.ts create mode 100644 packages/wotan/src/state-persistence.ts diff --git a/packages/wotan/src/dependency-resolver.ts b/packages/wotan/src/dependency-resolver.ts new file mode 100644 index 000000000..c4f81c5e9 --- /dev/null +++ b/packages/wotan/src/dependency-resolver.ts @@ -0,0 +1,227 @@ +import { injectable } from 'inversify'; +import * as ts from 'typescript'; +import { isModuleDeclaration, isNamespaceExportDeclaration, findImports, ImportKind } from 'tsutils'; +import { resolveCachedResult, getOutputFileNamesOfProjectReference, iterateProjectReferences } from './utils'; +import bind from 'bind-decorator'; + +export interface DependencyResolver { + update(program: DependencyResolverProgram, updatedFiles: Iterable): void; + getDependencies(fileName: string): ReadonlyArray; + getFilesAffectingGlobalScope(): ReadonlyArray; +} + +export type DependencyResolverProgram = + Pick; + +@injectable() +export class DependencyResolverFactory { + public create(host: ts.CompilerHost, program: DependencyResolverProgram): DependencyResolver { + return new DependencyResolverImpl(host, program); + } +} + +class DependencyResolverImpl implements DependencyResolver { + private affectsGlobalScope!: ReadonlyArray; + private ambientModules!: ReadonlyMap; + private moduleAugmentations!: ReadonlyMap; + private patternAmbientModules!: ReadonlyMap; + private ambientModuleAugmentations!: ReadonlyMap; + private patternModuleAugmentations!: ReadonlyMap; + private moduleDependenciesPerFile!: ReadonlyMap; + private dependencies = new Map>(); + private fileToProjectReference: ReadonlyMap | undefined = undefined; + + private cache = ts.createModuleResolutionCache(this.program.getCurrentDirectory(), (f) => this.host.getCanonicalFileName(f)); + constructor(private host: ts.CompilerHost, private program: DependencyResolverProgram) { + this.collectMetaData(); + } + + public update(program: DependencyResolverProgram, updatedFiles: Iterable) { + for (const file of updatedFiles) + this.dependencies.delete(file); + this.program = program; + this.collectMetaData(); + } + + private collectMetaData() { + const affectsGlobalScope = new Set(); + const ambientModules = new Map(); + const patternAmbientModules = new Map(); + const moduleAugmentationsTemp = new Map(); + const moduleDepencenciesPerFile = new Map(); + for (const file of this.program.getSourceFiles()) { + const meta = collectFileMetadata(file); + if (meta.affectsGlobalScope) + affectsGlobalScope.add(file.fileName); + for (const ambientModule of meta.ambientModules) { + const map = meta.isExternalModule + ? moduleAugmentationsTemp + : ambientModule.includes('*') + ? patternAmbientModules + : ambientModules; + addToListWithReverse(map, ambientModule, file.fileName, meta.isExternalModule ? undefined : moduleDepencenciesPerFile); + const existing = map.get(ambientModule); + if (existing === undefined) { + map.set(ambientModule, [file.fileName]); + } else { + existing.push(file.fileName); + } + } + } + + const ambientModuleAugmentations = new Map(); + const moduleAugmentations = new Map(); + const patternModuleAugmentations = new Map(); + for (const [module, files] of moduleAugmentationsTemp) { + if (ambientModules.has(module)) { + ambientModuleAugmentations.set(module, files); + continue; + } + for (const file of files) { + const {resolvedModule} = ts.resolveModuleName(module, file, this.program.getCompilerOptions(), this.host, this.cache); + if (resolvedModule !== undefined) { + addToListWithReverse(moduleAugmentations, resolvedModule.resolvedFileName, file, moduleDepencenciesPerFile); + } else { + const matchingPattern = getBestMatchingPattern(module, patternAmbientModules.keys()); + if (matchingPattern !== undefined) + addToListWithReverse(patternModuleAugmentations, matchingPattern, file, moduleDepencenciesPerFile); + } + } + } + + this.ambientModules = ambientModules; + this.patternAmbientModules = patternAmbientModules; + this.ambientModuleAugmentations = ambientModuleAugmentations; + this.moduleAugmentations = moduleAugmentations; + this.patternModuleAugmentations = patternModuleAugmentations; + this.moduleDependenciesPerFile = moduleDepencenciesPerFile; + this.affectsGlobalScope = Array.from(affectsGlobalScope).sort(); + } + + public getDependencies(file: string) { + const result = new Set(); + const dependenciesFromModuleDeclarations = this.moduleDependenciesPerFile.get(file); + if (dependenciesFromModuleDeclarations) + for (const deps of dependenciesFromModuleDeclarations) + addAllExceptSelf(result, deps, file); + addAllExceptSelf(result, resolveCachedResult(this.dependencies, file, this.resolveDependencies), file); + return Array.from(result).sort(); + } + + @bind + private resolveDependencies(fileName: string) { + const result = new Set(); + const sourceFile = this.program.getSourceFile(fileName)!; + let redirect: ts.ResolvedProjectReference | undefined; + let options: ts.CompilerOptions | undefined; + for (const {text: moduleName} of findImports(sourceFile, ImportKind.All)) { + const filesAffectingAmbientModule = this.ambientModules.get(moduleName); + if (filesAffectingAmbientModule !== undefined) { + addAllExceptSelf(result, filesAffectingAmbientModule, moduleName); + addAllExceptSelf(result, this.ambientModuleAugmentations.get(moduleName), fileName); + continue; + } + + if (options === undefined) { + if (this.fileToProjectReference === undefined) + this.fileToProjectReference = createProjectReferenceMap(this.program.getResolvedProjectReferences()); + redirect = this.fileToProjectReference.get(fileName); + options = redirect === undefined ? this.program.getCompilerOptions() : redirect.commandLine.options; + } + + const {resolvedModule} = ts.resolveModuleName(moduleName, fileName, options, this.host, this.cache, redirect); + if (resolvedModule !== undefined) { + if (resolvedModule.resolvedFileName !== fileName) + result.add(resolvedModule.resolvedFileName); + addAllExceptSelf(result, this.moduleAugmentations.get(resolvedModule.resolvedFileName), fileName); + } else { + const pattern = getBestMatchingPattern(moduleName, this.patternAmbientModules.keys()); + if (pattern !== undefined) { + addAllExceptSelf(result, this.patternAmbientModules.get(pattern), fileName); + addAllExceptSelf(result, this.patternModuleAugmentations.get(pattern), fileName); + } + } + } + + return result; + } + + public getFilesAffectingGlobalScope() { + return this.affectsGlobalScope; + } +} + +function getBestMatchingPattern(moduleName: string, patternAmbientModules: Iterable) { + // TypeScript uses the pattern with the longest matching prefix + let longestMatchLength = -1; + let longestMatch: string | undefined; + for (const pattern of patternAmbientModules) { + if (moduleName.length < pattern.length - 1) + continue; // compare length without the wildcard first, to avoid false positives like 'foo' matching 'foo*oo' + const index = pattern.indexOf('*'); + if ( + index > longestMatchLength && + moduleName.startsWith(pattern.substring(0, index)) && + moduleName.endsWith(pattern.substring(index + 1)) + ) { + longestMatchLength = index; + longestMatch = pattern; + } + } + return longestMatch; +} + +function addAllExceptSelf(receiver: Set, list: Iterable | undefined, current: string) { + if (list) + for (const file of list) + if (file !== current) + receiver.add(file); +} + +function createProjectReferenceMap(references: ts.ResolvedProjectReference['references']) { + const result = new Map(); + for (const ref of iterateProjectReferences(references)) + for (const file of getOutputFileNamesOfProjectReference(ref)) + result.set(file, ref); + return result; +} + +function addToListWithReverse(map: Map, key: string, value: string, reverse?: Map) { + const list = addToList(map, key, value); + if (reverse !== undefined) + addToList(reverse, value, list); +} + +function addToList(map: Map, key: string, value: T) { + let arr = map.get(key); + if (arr === undefined) { + map.set(key, arr = [value]); + } else { + arr.push(value); + } + return arr; +} + +interface MetaData { + affectsGlobalScope: boolean; + ambientModules: Set; + isExternalModule: boolean; +} + +function collectFileMetadata(sourceFile: ts.SourceFile): MetaData { + let affectsGlobalScope: boolean | undefined; + const ambientModules = new Set(); + const isExternalModule = ts.isExternalModule(sourceFile); + for (const statement of sourceFile.statements) { + if (statement.flags & ts.NodeFlags.GlobalAugmentation) { + affectsGlobalScope = true; + } else if (isModuleDeclaration(statement) && statement.name.kind === ts.SyntaxKind.StringLiteral) { + ambientModules.add(statement.name.text); + } else if (isNamespaceExportDeclaration(statement)) { + affectsGlobalScope = true; // TODO that's only correct with allowUmdGlobalAccess compilerOption + } else if (affectsGlobalScope === undefined) { // files that only consist of ambient modules do not affect global scope + affectsGlobalScope = !isExternalModule; + } + } + return {ambientModules, isExternalModule, affectsGlobalScope: affectsGlobalScope === true}; +} diff --git a/packages/wotan/src/linter.ts b/packages/wotan/src/linter.ts index 622d13cf1..a1f2699ee 100644 --- a/packages/wotan/src/linter.ts +++ b/packages/wotan/src/linter.ts @@ -18,7 +18,7 @@ import { applyFixes } from './fix'; import * as debug from 'debug'; import { injectable } from 'inversify'; import { RuleLoader } from './services/rule-loader'; -import { calculateChangeRange, invertChangeRange } from './utils'; +import { calculateChangeRange, invertChangeRange, mapDefined } from './utils'; import { ConvertedAst, convertAst, isCompilerOptionEnabled, getCheckJsDirective } from 'tsutils'; const log = debug('wotan:linter'); @@ -108,13 +108,14 @@ export class Linter { programFactory?: ProgramFactory, processor?: AbstractProcessor, options: LinterOptions = {}, + /** Initial set of findings from a cache. If provided, the initial linting is skipped and these findings are used for fixing. */ + findings = this.getFindings(file, config, programFactory, processor, options), ): LintAndFixFileResult { let totalFixes = 0; - let findings = this.getFindings(file, config, programFactory, processor, options); for (let i = 0; i < iterations; ++i) { if (findings.length === 0) break; - const fixes = findings.map((f) => f.fix).filter((f: T | undefined): f is T => f !== undefined); + const fixes = mapDefined(findings, (f) => f.fix); if (fixes.length === 0) { log('No fixes'); break; diff --git a/packages/wotan/src/program-state.ts b/packages/wotan/src/program-state.ts new file mode 100644 index 000000000..bcc9c3ae3 --- /dev/null +++ b/packages/wotan/src/program-state.ts @@ -0,0 +1,206 @@ +import { injectable } from 'inversify'; +import * as ts from 'typescript'; +import { DependencyResolver, DependencyResolverFactory, DependencyResolverProgram } from './dependency-resolver'; +import { resolveCachedResult, djb2, arraysAreEqual } from './utils'; +import bind from 'bind-decorator'; +import { Finding, ReducedConfiguration } from '@fimbul/ymir'; +import debug = require('debug'); + +const log = debug('wotan:programState'); + +export interface StaticProgramState { + optionsHash: string; + filesAffectingGlobalScope: ReadonlyArray; + files: Record, + result?: ReadonlyArray, + configHash?: string, + }>; +} + +@injectable() +export class ProgramStateFactory { + constructor(private resolverFactory: DependencyResolverFactory) {} + + public create(program: DependencyResolverProgram, host: ts.CompilerHost) { + return new ProgramState(program, this.resolverFactory.create(host, program)); + } +} + +class ProgramState { + private optionsHash = computeCompilerOptionsHash(this.program.getCompilerOptions()); + private fileHashes = new Map(); + private knownOutdated: boolean | undefined = undefined; + private fileResults = new Map}>(); + constructor(private program: DependencyResolverProgram, private resolver: DependencyResolver) {} + + public update(program: DependencyResolverProgram, updatedFiles: Iterable) { + this.knownOutdated = undefined; + this.resolver.update(program, updatedFiles); + for (const file of updatedFiles) + this.fileHashes.delete(file); + } + + private getFileHash(file: string) { + return resolveCachedResult(this.fileHashes, file, this.computeFileHash); + } + + @bind + private computeFileHash(file: string) { + return '' + djb2(this.program.getSourceFile(file)!.text); + } + + public getUpToDateResult(fileName: string, config: ReducedConfiguration, oldState?: StaticProgramState) { + if (!this.isUpToDate(fileName, config, oldState)) + return; + log('reusing state for %s', fileName); + return oldState!.files[fileName].result; + } + + public setFileResult(fileName: string, config: ReducedConfiguration, result: ReadonlyArray) { + this.fileResults.set(fileName, {result, configHash: '' + djb2(JSON.stringify(config))}); // TODO absolute paths + } + + public isUpToDate(fileName: string, config: ReducedConfiguration, oldState?: StaticProgramState) { + if (oldState === undefined) + return false; + if (this.knownOutdated === undefined) + this.knownOutdated = this.optionsHash !== oldState.optionsHash || + !arraysAreEqual(oldState.filesAffectingGlobalScope, this.resolver.getFilesAffectingGlobalScope()); + if (this.knownOutdated) + return false; + const old = oldState.files[fileName]; + if (old === undefined || old.result === undefined || old.configHash !== '' + djb2(JSON.stringify(config))) // TODO config contains absolute paths + return false; + return this.isFileUpToDate(fileName, oldState, new Set()); + } + + private isFileUpToDate(fileName: string, oldState: StaticProgramState, seen: Set) { + seen.add(fileName); + const old = oldState.files[fileName]; // TODO use relative file names? + if (old === undefined || old.hash !== this.getFileHash(fileName)) + return false; + const dependencies = this.resolver.getDependencies(fileName); + if (!arraysAreEqual(dependencies, old.dependencies)) + return false; + for (const dep of dependencies) + if (!seen.has(dep) && !this.isFileUpToDate(dep, oldState, seen)) + return false; + return true; + } + + public aggregate(oldState?: StaticProgramState): StaticProgramState { + const files: StaticProgramState['files'] = {}; + for (const file of this.program.getSourceFiles()) + files[file.fileName] = { + ...oldState && oldState.files[file.fileName], + hash: this.getFileHash(file.fileName), + dependencies: this.resolver.getDependencies(file.fileName), + ...this.fileResults.get(file.fileName), + }; + return { + files, + optionsHash: this.optionsHash, + filesAffectingGlobalScope: this.resolver.getFilesAffectingGlobalScope(), + }; + } +} + +function computeCompilerOptionsHash(options: ts.CompilerOptions) { + const obj: Record = {}; + for (const key of Object.keys(options).sort()) + if (isKnownCompilerOption(key)) + obj[key] = options[key]; + return '' + djb2(JSON.stringify(obj)); +} + +function isKnownCompilerOption(option: string): boolean { + type KnownOptions = + {[K in keyof ts.CompilerOptions]: string extends K ? never : K} extends {[K in keyof ts.CompilerOptions]: infer P} ? P : never; + const o = option; + switch (o) { + case 'allowJs': // + case 'allowSyntheticDefaultImports': + case 'allowUnreachableCode': + case 'allowUnusedLabels': + case 'alwaysStrict': + case 'baseUrl': + case 'charset': + case 'checkJs': + case 'composite': + case 'declaration': + case 'declarationDir': // + case 'declarationMap': // + case 'disableSizeLimit': // + case 'downlevelIteration': + case 'emitBOM': // + case 'emitDeclarationOnly': // + case 'emitDecoratorMetadata': + case 'esModuleInterop': + case 'experimentalDecorators': + case 'forceConsistentCasingInFileNames': + case 'importHelpers': + case 'incremental': // + case 'inlineSourceMap': // + case 'inlineSources': // + case 'isolatedModules': + case 'jsx': + case 'jsxFactory': + case 'keyofStringsOnly': + case 'lib': + case 'locale': + case 'mapRoot': + case 'maxNodeModuleJsDepth': + case 'module': + case 'moduleResolution': + case 'newLine': // + case 'noEmit': // + case 'noEmitHelpers': // + case 'noEmitOnError': // + case 'noErrorTruncation': + case 'noFallthroughCasesInSwitch': + case 'noImplicitAny': + case 'noImplicitReturns': + case 'noImplicitThis': + case 'noImplicitUseStrict': + case 'noLib': + case 'noResolve': + case 'noStrictGenericChecks': + case 'noUnusedLocals': + case 'noUnusedParameters': + case 'out': // + case 'outDir': // + case 'outFile': // + case 'paths': + case 'preserveConstEnums': // + case 'preserveSymlinks': + case 'project': // + case 'reactNamespace': + case 'removeComments': // + case 'resolveJsonModule': + case 'rootDir': + case 'rootDirs': + case 'skipDefaultLibCheck': + case 'skipLibCheck': + case 'sourceMap': // + case 'sourceRoot': // + case 'strict': + case 'strictBindCallApply': + case 'strictFunctionTypes': + case 'strictNullChecks': + case 'strictPropertyInitialization': + case 'stripInternal': + case 'suppressExcessPropertyErrors': + case 'suppressImplicitAnyIndexErrors': + case 'target': + case 'traceResolution': // + case 'tsBuildInfoFile': // + case 'typeRoots': + case 'types': + return true; + default: + type AssertNever = T; + return >false; + } +} diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index 3c961eec7..aa87958b3 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -22,6 +22,8 @@ import { ConfigurationManager } from './services/configuration-manager'; import { ProjectHost } from './project-host'; import debug = require('debug'); import { normalizeGlob } from 'normalize-glob'; +import { ProgramStateFactory } from './program-state'; +import { StatePersistence } from './state-persistence'; const log = debug('wotan:runner'); @@ -34,6 +36,7 @@ export interface LintOptions { fix: boolean | number; extensions: ReadonlyArray | undefined; reportUselessDirectives: Severity | boolean | undefined; + cache: boolean; } interface NormalizedOptions extends Pick> { @@ -55,6 +58,8 @@ export class Runner { private directories: DirectoryService, private logger: MessageHandler, private filterFactory: FileFilterFactory, + private programStateFactory: ProgramStateFactory, + private statePersistence: StatePersistence, ) {} public lintCollection(options: LintOptions): LintResult { @@ -85,10 +90,13 @@ export class Runner { this.configManager, this.processorLoader, ); - for (let {files, program} of + for (let {files, program, configFilePath: tsconfigPath} of this.getFilesAndProgram(options.project, options.files, options.exclude, processorHost, options.references) ) { + const oldState = options.cache ? this.statePersistence.loadState(tsconfigPath) : undefined; + const programState = options.cache ? this.programStateFactory.create(program, processorHost) : undefined; let invalidatedProgram = false; + let updatedFiles: string[] = []; const factory: ProgramFactory = { getCompilerOptions() { return program.getCompilerOptions(); @@ -110,6 +118,11 @@ export class Runner { const effectiveConfig = config && this.configManager.reduce(config, originalName); if (effectiveConfig === undefined) continue; + if (programState !== undefined && updatedFiles.length !== 0) { + // TODO this is not correct as Program still contains the old SourceFile + programState.update(program, updatedFiles); + updatedFiles = []; + } let sourceFile = program.getSourceFile(file)!; const originalContent = mapped === undefined ? sourceFile.text : mapped.originalContent; let summary: FileSummary; @@ -120,13 +133,14 @@ export class Runner { originalContent, effectiveConfig, (content, range) => { - invalidatedProgram = true; const oldContent = sourceFile.text; sourceFile = ts.updateSourceFile(sourceFile, content, range); const hasErrors = hasParseErrors(sourceFile); if (hasErrors) { log("Autofixing caused syntax errors in '%s', rolling back", sourceFile.fileName); sourceFile = ts.updateSourceFile(sourceFile, oldContent, invertChangeRange(range)); + } else { + updatedFiles.push(sourceFile.fileName); } // either way we need to store the new SourceFile as the old one is now corrupted processorHost.updateSourceFile(sourceFile); @@ -136,22 +150,29 @@ export class Runner { factory, mapped === undefined ? undefined : mapped.processor, linterOptions, + // pass cached results so we can apply fixes from cache + programState && programState.getUpToDateResult(file, effectiveConfig, oldState), ); } else { summary = { - findings: this.linter.getFindings( - sourceFile, - effectiveConfig, - factory, - mapped === undefined ? undefined : mapped.processor, - linterOptions, - ), + findings: programState && programState.getUpToDateResult(file, effectiveConfig, oldState) || + this.linter.getFindings( + sourceFile, + effectiveConfig, + factory, + mapped === undefined ? undefined : mapped.processor, + linterOptions, + ), fixes: 0, content: originalContent, }; } + if (programState !== undefined) + programState.setFileResult(file, effectiveConfig, summary.findings); yield [originalName, summary]; } + if (programState !== undefined) + this.statePersistence.saveState(tsconfigPath, programState.aggregate(oldState)); } } @@ -241,7 +262,7 @@ export class Runner { exclude: ReadonlyArray, host: ProjectHost, references: boolean, - ): Iterable<{files: Iterable, program: ts.Program}> { + ): Iterable<{files: Iterable, program: ts.Program, configFilePath: string}> { const cwd = unixifyPath(this.directories.getCurrentDirectory()); if (projects.length !== 0) { projects = projects.map((configFile) => this.checkConfigDirectory(unixifyPath(path.resolve(cwd, configFile)))); @@ -265,7 +286,7 @@ export class Runner { const ex = exclude.map((p) => new Minimatch(p, {dot: true})); const projectsSeen: string[] = []; let filesOfPreviousProject: string[] | undefined; - for (const program of this.createPrograms(projects, host, projectsSeen, references, isFileIncluded)) { + for (const {program, configFilePath} of this.createPrograms(projects, host, projectsSeen, references, isFileIncluded)) { const ownFiles = []; const files: string[] = []; const fileFilter = this.filterFactory.create({program, host}); @@ -289,7 +310,7 @@ export class Runner { filesOfPreviousProject = ownFiles; if (files.length !== 0) - yield {files, program}; + yield {files, program, configFilePath}; } ensurePatternsMatch(nonMagicGlobs, ex, allMatchedFiles, projectsSeen); @@ -319,7 +340,7 @@ export class Runner { seen: string[], references: boolean, isFileIncluded: (fileName: string) => boolean, - ): Iterable { + ): Iterable<{program: ts.Program, configFilePath: string}> { for (const configFile of projects) { if (configFile === undefined) continue; @@ -352,7 +373,7 @@ export class Runner { // this is in a nested block to allow garbage collection while recursing const program = host.createProgram(commandLine.fileNames, commandLine.options, undefined, commandLine.projectReferences); - yield program; + yield {program, configFilePath}; if (references) resolvedReferences = program.getResolvedProjectReferences(); } diff --git a/packages/wotan/src/state-persistence.ts b/packages/wotan/src/state-persistence.ts new file mode 100644 index 000000000..d4460ae4a --- /dev/null +++ b/packages/wotan/src/state-persistence.ts @@ -0,0 +1,45 @@ +import { injectable } from 'inversify'; +import { CachedFileSystem } from './services/cached-file-system'; +import { StaticProgramState } from './program-state'; +import { safeLoad, safeDump } from 'js-yaml'; +import debug = require('debug'); +import { DirectoryService } from '@fimbul/ymir'; + +const log = debug('wotan:statePersistence'); + +interface CacheFileContent { + version: string; + projects: Record; +} + +const CACHE_VERSION = '1'; + +@injectable() +export class StatePersistence { + constructor(private fs: CachedFileSystem, private dir: DirectoryService) {} + + public loadState(project: string): StaticProgramState | undefined { + const content = this.loadExisting(); + return content && content.projects[project]; // TODO resolve all paths + } + + public saveState(project: string, state: StaticProgramState) { + const content = this.loadExisting() || {version: CACHE_VERSION, projects: {}}; + content.projects[project] = state; // TODO make all paths relative + this.fs.writeFile(this.dir.getCurrentDirectory() + '/.fimbullintercache.yaml', safeDump(content, {indent: 2, sortKeys: true})); + } + + private loadExisting(): CacheFileContent | undefined { + const fileName = this.dir.getCurrentDirectory() + '/.fimbullintercache.yaml'; + if (!this.fs.isFile(fileName)) { + log('%s does not exist', fileName); + return; + } + const content = safeLoad(this.fs.readFile(fileName))!; + if (content.version !== CACHE_VERSION) { + log('cache version mismatch'); + return; + } + return content; + } +} diff --git a/packages/wotan/src/utils.ts b/packages/wotan/src/utils.ts index aa054bf08..ae6225fc3 100644 --- a/packages/wotan/src/utils.ts +++ b/packages/wotan/src/utils.ts @@ -225,3 +225,19 @@ function getOutFileDeclarationName(outFile: string) { // outFile ignores declarationDir return outFile.slice(0, -path.extname(outFile).length) + '.d.ts'; } + +export function djb2(str: string) { + let hash = 5381; + for (let i = 0; i < str.length; ++i) + hash = ((hash << 5) + hash) + str.charCodeAt(i); + return hash; +} + +export function arraysAreEqual(a: ReadonlyArray, b: ReadonlyArray) { + if (a.length !== b.length) + return false; + for (let i = 0; i < a.length; ++i) + if (a[i] !== b[i]) + return false; + return true; +} From 2b4a704f2a6dc5e081a0f1a1110032b2cdc28bff Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Mon, 25 Jan 2021 19:35:45 +0100 Subject: [PATCH 02/37] wip --- .fimbullinter.yaml | 1 + packages/bifrost/package.json | 2 +- packages/disir/package.json | 2 +- packages/heimdall/package.json | 3 +- packages/mithotyn/package.json | 3 +- packages/valtyr/package.json | 2 +- packages/ve/package.json | 1 + packages/wotan/package.json | 2 +- packages/wotan/src/argparse.ts | 4 + packages/wotan/src/dependency-resolver.ts | 212 ++++++----- packages/wotan/src/di/core.module.ts | 6 + packages/wotan/src/program-state.ts | 426 +++++++++++++++++----- packages/wotan/src/runner.ts | 34 +- packages/wotan/src/state-persistence.ts | 53 +-- packages/wotan/src/utils.ts | 9 - packages/ymir/package.json | 3 +- yarn.lock | 13 +- 17 files changed, 517 insertions(+), 259 deletions(-) diff --git a/.fimbullinter.yaml b/.fimbullinter.yaml index 4ddb30663..b66b1fc53 100644 --- a/.fimbullinter.yaml +++ b/.fimbullinter.yaml @@ -1,3 +1,4 @@ project: - tsconfig.json reportUselessDirectives: true +cache: true diff --git a/packages/bifrost/package.json b/packages/bifrost/package.json index 41c6bbf89..796e5af12 100644 --- a/packages/bifrost/package.json +++ b/packages/bifrost/package.json @@ -33,7 +33,7 @@ "@fimbul/ymir": "^0.22.0", "get-caller-file": "^2.0.0", "tslib": "^2.0.0", - "tsutils": "^3.5.0" + "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" }, "devDependencies": { "@types/get-caller-file": "^1.0.0" diff --git a/packages/disir/package.json b/packages/disir/package.json index 2be4cea72..4b7984e67 100644 --- a/packages/disir/package.json +++ b/packages/disir/package.json @@ -7,7 +7,7 @@ "dependencies": { "@fimbul/ymir": "^0.22.0", "tslib": "^2.0.0", - "tsutils": "^3.5.0" + "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" }, "peerDependencies": { "typescript": ">= 4.0.0 || >= 4.2.0-dev" diff --git a/packages/heimdall/package.json b/packages/heimdall/package.json index f9c9d7e09..c05451660 100644 --- a/packages/heimdall/package.json +++ b/packages/heimdall/package.json @@ -33,6 +33,7 @@ "@fimbul/bifrost": "^0.22.0", "inversify": "^5.0.0", "tslib": "^2.0.0", - "tslint": "^5.0.0" + "tslint": "^5.0.0", + "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" } } diff --git a/packages/mithotyn/package.json b/packages/mithotyn/package.json index 58764f78a..99aced50f 100644 --- a/packages/mithotyn/package.json +++ b/packages/mithotyn/package.json @@ -32,7 +32,8 @@ "editor" ], "dependencies": { - "mock-require": "^3.0.2" + "mock-require": "^3.0.2", + "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" }, "devDependencies": { "@fimbul/wotan": "^0.16.0", diff --git a/packages/valtyr/package.json b/packages/valtyr/package.json index 0f33c8533..d46e58637 100644 --- a/packages/valtyr/package.json +++ b/packages/valtyr/package.json @@ -39,6 +39,6 @@ "inversify": "^5.0.0", "tslib": "^2.0.0", "tslint": "^5.0.0", - "tsutils": "^3.5.0" + "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" } } diff --git a/packages/ve/package.json b/packages/ve/package.json index 6eb97fe1c..23533c5f2 100644 --- a/packages/ve/package.json +++ b/packages/ve/package.json @@ -32,6 +32,7 @@ "@fimbul/ymir": "^0.22.0", "parse5-sax-parser": "^6.0.0", "tslib": "^2.0.0", + "tsutils": "../../../tsutils/tsutils-3.19.1.tgz", "void-elements": "^3.1.0" }, "devDependencies": { diff --git a/packages/wotan/package.json b/packages/wotan/package.json index ad1f90796..518715ed1 100644 --- a/packages/wotan/package.json +++ b/packages/wotan/package.json @@ -59,7 +59,7 @@ "semver": "^7.0.0", "stable": "^0.1.8", "tslib": "^2.0.0", - "tsutils": "^3.18.0" + "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" }, "peerDependencies": { "typescript": ">= 4.0.0 || >= 4.2.0-dev" diff --git a/packages/wotan/src/argparse.ts b/packages/wotan/src/argparse.ts index a7fc36238..dc65ec02e 100644 --- a/packages/wotan/src/argparse.ts +++ b/packages/wotan/src/argparse.ts @@ -53,6 +53,7 @@ export const GLOBAL_OPTIONS_SPEC = { exclude: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), []), project: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), []), references: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitive('boolean'), false), + cache: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitive('boolean'), false), formatter: OptionParser.Factory.parsePrimitive('string'), fix: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitive('boolean', 'number'), false), extensions: OptionParser.Transform.map(OptionParser.Factory.parsePrimitiveOrArray('string'), sanitizeExtensionArgument), @@ -116,6 +117,9 @@ function parseLintCommand( case '--references': ({index: i, argument: result.references} = parseOptionalBoolean(args, i)); break; + case '--cache': + ({index: i, argument: result.cache} = parseOptionalBoolean(args, i)); + break; case '-e': case '--exclude': result.exclude = exclude; diff --git a/packages/wotan/src/dependency-resolver.ts b/packages/wotan/src/dependency-resolver.ts index c4f81c5e9..59a60728a 100644 --- a/packages/wotan/src/dependency-resolver.ts +++ b/packages/wotan/src/dependency-resolver.ts @@ -1,153 +1,164 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; -import { isModuleDeclaration, isNamespaceExportDeclaration, findImports, ImportKind } from 'tsutils'; +import { isModuleDeclaration, isNamespaceExportDeclaration, findImports, ImportKind, isCompilerOptionEnabled } from 'tsutils'; import { resolveCachedResult, getOutputFileNamesOfProjectReference, iterateProjectReferences } from './utils'; import bind from 'bind-decorator'; +import { ProjectHost } from './project-host'; export interface DependencyResolver { - update(program: DependencyResolverProgram, updatedFiles: Iterable): void; - getDependencies(fileName: string): ReadonlyArray; - getFilesAffectingGlobalScope(): ReadonlyArray; + update(program: ts.Program, updatedFile: string): void; + getDependencies(fileName: string): ReadonlyMap; + getFilesAffectingGlobalScope(): readonly string[]; } -export type DependencyResolverProgram = - Pick; - @injectable() export class DependencyResolverFactory { - public create(host: ts.CompilerHost, program: DependencyResolverProgram): DependencyResolver { + public create(host: ProjectHost, program: ts.Program): DependencyResolver { return new DependencyResolverImpl(host, program); } } +export interface DependencyResolverState { + affectsGlobalScope: readonly string[]; + ambientModules: ReadonlyMap; + moduleAugmentations: ReadonlyMap; + patternAmbientModules: ReadonlyMap; +} + class DependencyResolverImpl implements DependencyResolver { - private affectsGlobalScope!: ReadonlyArray; - private ambientModules!: ReadonlyMap; - private moduleAugmentations!: ReadonlyMap; - private patternAmbientModules!: ReadonlyMap; - private ambientModuleAugmentations!: ReadonlyMap; - private patternModuleAugmentations!: ReadonlyMap; - private moduleDependenciesPerFile!: ReadonlyMap; - private dependencies = new Map>(); + private dependencies = new Map>(); private fileToProjectReference: ReadonlyMap | undefined = undefined; + private fileMetadata = new Map(); + private compilerOptions = this.program.getCompilerOptions(); - private cache = ts.createModuleResolutionCache(this.program.getCurrentDirectory(), (f) => this.host.getCanonicalFileName(f)); - constructor(private host: ts.CompilerHost, private program: DependencyResolverProgram) { - this.collectMetaData(); - } + private state: DependencyResolverState | undefined = undefined; - public update(program: DependencyResolverProgram, updatedFiles: Iterable) { - for (const file of updatedFiles) - this.dependencies.delete(file); + constructor(private host: ProjectHost, private program: ts.Program) {} + + public update(program: ts.Program, updatedFile: string) { + this.state = undefined; + this.dependencies.delete(updatedFile); + this.fileMetadata.delete(updatedFile); this.program = program; - this.collectMetaData(); } - private collectMetaData() { - const affectsGlobalScope = new Set(); + private buildState(): DependencyResolverState { + const affectsGlobalScope = []; const ambientModules = new Map(); const patternAmbientModules = new Map(); const moduleAugmentationsTemp = new Map(); - const moduleDepencenciesPerFile = new Map(); for (const file of this.program.getSourceFiles()) { - const meta = collectFileMetadata(file); + const meta = this.getFileMetaData(file.fileName); if (meta.affectsGlobalScope) - affectsGlobalScope.add(file.fileName); + affectsGlobalScope.push(file.fileName); for (const ambientModule of meta.ambientModules) { const map = meta.isExternalModule ? moduleAugmentationsTemp : ambientModule.includes('*') ? patternAmbientModules : ambientModules; - addToListWithReverse(map, ambientModule, file.fileName, meta.isExternalModule ? undefined : moduleDepencenciesPerFile); - const existing = map.get(ambientModule); - if (existing === undefined) { - map.set(ambientModule, [file.fileName]); - } else { - existing.push(file.fileName); - } + addToList(map, ambientModule, file.fileName); } } - const ambientModuleAugmentations = new Map(); const moduleAugmentations = new Map(); - const patternModuleAugmentations = new Map(); for (const [module, files] of moduleAugmentationsTemp) { - if (ambientModules.has(module)) { - ambientModuleAugmentations.set(module, files); + // if an ambient module with the same identifier exists, the augmentation always applies to that + const ambientModuleAffectingFiles = ambientModules.get(module); + if (ambientModuleAffectingFiles !== undefined) { + ambientModuleAffectingFiles.push(...files); continue; } for (const file of files) { - const {resolvedModule} = ts.resolveModuleName(module, file, this.program.getCompilerOptions(), this.host, this.cache); - if (resolvedModule !== undefined) { - addToListWithReverse(moduleAugmentations, resolvedModule.resolvedFileName, file, moduleDepencenciesPerFile); + const resolved = this.getExternalReferences(file).get(module); + // if an augmentation's identifier can be resolved from the declaring file, the augmentation applies to the resolved path + if (resolved != null) { + addToList(moduleAugmentations, resolved, file); } else { + // if a pattern ambient module matches the augmented identifier, the augmentation applies to that const matchingPattern = getBestMatchingPattern(module, patternAmbientModules.keys()); if (matchingPattern !== undefined) - addToListWithReverse(patternModuleAugmentations, matchingPattern, file, moduleDepencenciesPerFile); + addToList(patternAmbientModules, matchingPattern, file); } } } - this.ambientModules = ambientModules; - this.patternAmbientModules = patternAmbientModules; - this.ambientModuleAugmentations = ambientModuleAugmentations; - this.moduleAugmentations = moduleAugmentations; - this.patternModuleAugmentations = patternModuleAugmentations; - this.moduleDependenciesPerFile = moduleDepencenciesPerFile; - this.affectsGlobalScope = Array.from(affectsGlobalScope).sort(); + return { + affectsGlobalScope, + ambientModules, + moduleAugmentations, + patternAmbientModules, + }; } - public getDependencies(file: string) { - const result = new Set(); - const dependenciesFromModuleDeclarations = this.moduleDependenciesPerFile.get(file); - if (dependenciesFromModuleDeclarations) - for (const deps of dependenciesFromModuleDeclarations) - addAllExceptSelf(result, deps, file); - addAllExceptSelf(result, resolveCachedResult(this.dependencies, file, this.resolveDependencies), file); - return Array.from(result).sort(); + public getFilesAffectingGlobalScope() { + return (this.state ??= this.buildState()).affectsGlobalScope; } - @bind - private resolveDependencies(fileName: string) { - const result = new Set(); - const sourceFile = this.program.getSourceFile(fileName)!; - let redirect: ts.ResolvedProjectReference | undefined; - let options: ts.CompilerOptions | undefined; - for (const {text: moduleName} of findImports(sourceFile, ImportKind.All)) { - const filesAffectingAmbientModule = this.ambientModules.get(moduleName); + public getDependencies(file: string) { // TODO is it worth caching this? + this.state ??= this.buildState(); + const result = new Map(); + for (const [identifier, resolved] of this.getExternalReferences(file)) { + const filesAffectingAmbientModule = this.state.ambientModules.get(identifier); if (filesAffectingAmbientModule !== undefined) { - addAllExceptSelf(result, filesAffectingAmbientModule, moduleName); - addAllExceptSelf(result, this.ambientModuleAugmentations.get(moduleName), fileName); - continue; - } - - if (options === undefined) { - if (this.fileToProjectReference === undefined) - this.fileToProjectReference = createProjectReferenceMap(this.program.getResolvedProjectReferences()); - redirect = this.fileToProjectReference.get(fileName); - options = redirect === undefined ? this.program.getCompilerOptions() : redirect.commandLine.options; - } - - const {resolvedModule} = ts.resolveModuleName(moduleName, fileName, options, this.host, this.cache, redirect); - if (resolvedModule !== undefined) { - if (resolvedModule.resolvedFileName !== fileName) - result.add(resolvedModule.resolvedFileName); - addAllExceptSelf(result, this.moduleAugmentations.get(resolvedModule.resolvedFileName), fileName); + result.set(identifier, filesAffectingAmbientModule); + } else if (resolved !== null) { + const list = [resolved]; + const augmentations = this.state.moduleAugmentations.get(resolved); + if (augmentations !== undefined) + list.push(...augmentations); + result.set(identifier, list); } else { - const pattern = getBestMatchingPattern(moduleName, this.patternAmbientModules.keys()); + const pattern = getBestMatchingPattern(identifier, this.state.patternAmbientModules.keys()); if (pattern !== undefined) { - addAllExceptSelf(result, this.patternAmbientModules.get(pattern), fileName); - addAllExceptSelf(result, this.patternModuleAugmentations.get(pattern), fileName); + result.set(identifier, this.state.patternAmbientModules.get(pattern)!); + } else { + result.set(identifier, null); } } } + const meta = this.fileMetadata.get(file)!; + if (!meta.isExternalModule) + for (const ambientModule of meta.ambientModules) + result.set(ambientModule, this.state[ambientModule.includes('*') ? 'patternAmbientModules' : 'ambientModules'].get(ambientModule)!); return result; } - public getFilesAffectingGlobalScope() { - return this.affectsGlobalScope; + private getFileMetaData(fileName: string) { + return resolveCachedResult(this.fileMetadata, fileName, this.collectMetaDataForFile); + } + + private getExternalReferences(fileName: string) { + return resolveCachedResult(this.dependencies, fileName, this.collectExternalReferences); + } + + @bind + private collectMetaDataForFile(fileName: string) { + return collectFileMetadata(this.program.getSourceFile(fileName)!, this.compilerOptions); + } + + @bind + private collectExternalReferences(fileName: string): Map { + // TODO useSourceOfProjectReferenceRedirect + // TODO referenced files + const sourceFile = this.program.getSourceFile(fileName)!; + const references = new Set(findImports(sourceFile, ImportKind.All, false).map(({text}) => text)); + if (ts.isExternalModule(sourceFile)) { + // if (isCompilerOptionEnabled(this.compilerOptions, 'importHelpers')) + // references.add('tslib') + for (const augmentation of this.getFileMetaData(fileName).ambientModules) + references.add(augmentation); + } + const result = new Map(); + if (references.size === 0) + return result; + this.fileToProjectReference ??= createProjectReferenceMap(this.program.getResolvedProjectReferences()); + const arr = Array.from(references); + const resolved = this.host.resolveModuleNames(arr, fileName, undefined, this.fileToProjectReference.get(fileName)); + for (let i = 0; i < resolved.length; ++i) + result.set(arr[i], resolved[i]?.resolvedFileName ?? null); + return result; } } @@ -171,13 +182,6 @@ function getBestMatchingPattern(moduleName: string, patternAmbientModules: Itera return longestMatch; } -function addAllExceptSelf(receiver: Set, list: Iterable | undefined, current: string) { - if (list) - for (const file of list) - if (file !== current) - receiver.add(file); -} - function createProjectReferenceMap(references: ts.ResolvedProjectReference['references']) { const result = new Map(); for (const ref of iterateProjectReferences(references)) @@ -186,20 +190,13 @@ function createProjectReferenceMap(references: ts.ResolvedProjectReference['refe return result; } -function addToListWithReverse(map: Map, key: string, value: string, reverse?: Map) { - const list = addToList(map, key, value); - if (reverse !== undefined) - addToList(reverse, value, list); -} - function addToList(map: Map, key: string, value: T) { - let arr = map.get(key); + const arr = map.get(key); if (arr === undefined) { - map.set(key, arr = [value]); + map.set(key, [value]); } else { arr.push(value); } - return arr; } interface MetaData { @@ -208,7 +205,7 @@ interface MetaData { isExternalModule: boolean; } -function collectFileMetadata(sourceFile: ts.SourceFile): MetaData { +function collectFileMetadata(sourceFile: ts.SourceFile, compilerOptions: ts.CompilerOptions): MetaData { let affectsGlobalScope: boolean | undefined; const ambientModules = new Set(); const isExternalModule = ts.isExternalModule(sourceFile); @@ -218,7 +215,8 @@ function collectFileMetadata(sourceFile: ts.SourceFile): MetaData { } else if (isModuleDeclaration(statement) && statement.name.kind === ts.SyntaxKind.StringLiteral) { ambientModules.add(statement.name.text); } else if (isNamespaceExportDeclaration(statement)) { - affectsGlobalScope = true; // TODO that's only correct with allowUmdGlobalAccess compilerOption + if (isCompilerOptionEnabled(compilerOptions, 'allowUmdGlobalAccess')) + affectsGlobalScope = true; } else if (affectsGlobalScope === undefined) { // files that only consist of ambient modules do not affect global scope affectsGlobalScope = !isExternalModule; } diff --git a/packages/wotan/src/di/core.module.ts b/packages/wotan/src/di/core.module.ts index 54a57abe2..aa4183e2e 100644 --- a/packages/wotan/src/di/core.module.ts +++ b/packages/wotan/src/di/core.module.ts @@ -7,6 +7,9 @@ import { Linter } from '../linter'; import { Runner } from '../runner'; import { ProcessorLoader } from '../services/processor-loader'; import { GlobalOptions } from '@fimbul/ymir'; +import { ProgramStateFactory } from '../program-state'; +import { StatePersistence } from '../state-persistence'; +import { DependencyResolverFactory } from '../dependency-resolver'; export function createCoreModule(globalOptions: GlobalOptions) { return new ContainerModule((bind) => { @@ -17,6 +20,9 @@ export function createCoreModule(globalOptions: GlobalOptions) { bind(ProcessorLoader).toSelf(); bind(Linter).toSelf(); bind(Runner).toSelf(); + bind(ProgramStateFactory).toSelf(); + bind(StatePersistence).toSelf(); // TODO allow overriding + bind(DependencyResolverFactory).toSelf(); bind(GlobalOptions).toConstantValue(globalOptions); }); } diff --git a/packages/wotan/src/program-state.ts b/packages/wotan/src/program-state.ts index bcc9c3ae3..75358e494 100644 --- a/packages/wotan/src/program-state.ts +++ b/packages/wotan/src/program-state.ts @@ -1,45 +1,91 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; -import { DependencyResolver, DependencyResolverFactory, DependencyResolverProgram } from './dependency-resolver'; -import { resolveCachedResult, djb2, arraysAreEqual } from './utils'; +import { DependencyResolver, DependencyResolverFactory } from './dependency-resolver'; +import { resolveCachedResult, djb2 } from './utils'; import bind from 'bind-decorator'; -import { Finding, ReducedConfiguration } from '@fimbul/ymir'; +import { EffectiveConfiguration, Finding, ReducedConfiguration } from '@fimbul/ymir'; import debug = require('debug'); +import { ProjectHost } from './project-host'; +import { isCompilerOptionEnabled } from 'tsutils'; +import { StatePersistence } from './state-persistence'; +import * as path from 'path'; const log = debug('wotan:programState'); export interface StaticProgramState { - optionsHash: string; - filesAffectingGlobalScope: ReadonlyArray; - files: Record, - result?: ReadonlyArray, - configHash?: string, - }>; + // TODO add linter version + /** TypeScript version */ + readonly ts: string; + /** Hash of compilerOptions */ + readonly options: string; + /** Maps filename to index in 'files' array */ + readonly lookup: Readonly>; + /** Index of files that affect global scope */ + readonly global: readonly number[]; + /** Information about all files in the program */ + readonly files: readonly StaticProgramState.FileState[]; +} + +export namespace StaticProgramState { + export interface FileState { + readonly hash: string; + readonly dependencies: Readonly>; + readonly result?: readonly Finding[]; + readonly config?: string; + } } @injectable() export class ProgramStateFactory { - constructor(private resolverFactory: DependencyResolverFactory) {} + constructor(private resolverFactory: DependencyResolverFactory, private statePersistence: StatePersistence) {} - public create(program: DependencyResolverProgram, host: ts.CompilerHost) { - return new ProgramState(program, this.resolverFactory.create(host, program)); + public create(program: ts.Program, host: ProjectHost, tsconfigPath: string) { + return new ProgramState(program, this.resolverFactory.create(host, program), this.statePersistence, tsconfigPath); } } class ProgramState { private optionsHash = computeCompilerOptionsHash(this.program.getCompilerOptions()); + private assumeChangesOnlyAffectDirectDependencies = + isCompilerOptionEnabled(this.program.getCompilerOptions(), 'assumeChangesOnlyAffectDirectDependencies'); private fileHashes = new Map(); - private knownOutdated: boolean | undefined = undefined; - private fileResults = new Map}>(); - constructor(private program: DependencyResolverProgram, private resolver: DependencyResolver) {} + private fileResults = new Map}>(); + private relativePathNames = new Map(); + private _oldState: StaticProgramState | undefined; + private recheckOldState = true; + private projectDirectory = path.posix.dirname(this.project); + private dependenciesUpToDate = new Map(); + + constructor(private program: ts.Program, private resolver: DependencyResolver, private statePersistence: StatePersistence, private project: string) { + const oldState = this.statePersistence.loadState(project); + this._oldState = (oldState?.ts !== ts.version || oldState.options !== this.optionsHash) ? undefined : oldState; + } + + /** get old state if global files didn't change */ + private tryReuseOldState() { + if (this._oldState === undefined || !this.recheckOldState) + return this._oldState; + const filesAffectingGlobalScope = this.resolver.getFilesAffectingGlobalScope(); + if (this._oldState.global.length !== filesAffectingGlobalScope.length) + return this._oldState = undefined; + const globalFilesWithHash = this.sortByHash(filesAffectingGlobalScope); + for (let i = 0; i < globalFilesWithHash.length; ++i) { + const index = this._oldState.global[i]; + if ( + globalFilesWithHash[i].hash !== this._oldState.files[index].hash || + !this.assumeChangesOnlyAffectDirectDependencies && !this.isFileUpToDate(globalFilesWithHash[i].fileName, index, this._oldState) + ) + return this._oldState = undefined; + } + this.recheckOldState = false; + return this._oldState; + } - public update(program: DependencyResolverProgram, updatedFiles: Iterable) { - this.knownOutdated = undefined; - this.resolver.update(program, updatedFiles); - for (const file of updatedFiles) - this.fileHashes.delete(file); + public update(program: ts.Program, updatedFile: string) { + this.resolver.update(program, updatedFile); + this.fileHashes.delete(updatedFile); + this.recheckOldState = true; + this.dependenciesUpToDate = new Map(); } private getFileHash(file: string) { @@ -51,67 +97,239 @@ class ProgramState { return '' + djb2(this.program.getSourceFile(file)!.text); } - public getUpToDateResult(fileName: string, config: ReducedConfiguration, oldState?: StaticProgramState) { - if (!this.isUpToDate(fileName, config, oldState)) + private getRelativePath(fileName: string) { + return resolveCachedResult(this.relativePathNames, fileName, this.makeRelativePath); + } + + @bind + private makeRelativePath(fileName: string) { + return path.posix.relative(this.projectDirectory, fileName); + } + + public getUpToDateResult(fileName: string, config: ReducedConfiguration) { + const oldState = this.tryReuseOldState(); + if (oldState === undefined) + return; + const relative = this.getRelativePath(fileName); + if (!(relative in oldState.lookup)) + return; + const index = oldState.lookup[relative]; + const old = oldState.files[index]; + if ( + old.result === undefined || + old.config !== '' + djb2(JSON.stringify(stripConfig(config))) || + old.hash !== this.getFileHash(fileName) || + !this.isFileUpToDate(fileName, index, oldState) + ) return; log('reusing state for %s', fileName); - return oldState!.files[fileName].result; + return old.result; } public setFileResult(fileName: string, config: ReducedConfiguration, result: ReadonlyArray) { - this.fileResults.set(fileName, {result, configHash: '' + djb2(JSON.stringify(config))}); // TODO absolute paths + this.fileResults.set(fileName, {result, config: '' + djb2(JSON.stringify(stripConfig(config)))}); } - public isUpToDate(fileName: string, config: ReducedConfiguration, oldState?: StaticProgramState) { - if (oldState === undefined) - return false; - if (this.knownOutdated === undefined) - this.knownOutdated = this.optionsHash !== oldState.optionsHash || - !arraysAreEqual(oldState.filesAffectingGlobalScope, this.resolver.getFilesAffectingGlobalScope()); - if (this.knownOutdated) - return false; - const old = oldState.files[fileName]; - if (old === undefined || old.result === undefined || old.configHash !== '' + djb2(JSON.stringify(config))) // TODO config contains absolute paths - return false; - return this.isFileUpToDate(fileName, oldState, new Set()); - } - - private isFileUpToDate(fileName: string, oldState: StaticProgramState, seen: Set) { - seen.add(fileName); - const old = oldState.files[fileName]; // TODO use relative file names? - if (old === undefined || old.hash !== this.getFileHash(fileName)) - return false; - const dependencies = this.resolver.getDependencies(fileName); - if (!arraysAreEqual(dependencies, old.dependencies)) - return false; - for (const dep of dependencies) - if (!seen.has(dep) && !this.isFileUpToDate(dep, oldState, seen)) - return false; - return true; - } - - public aggregate(oldState?: StaticProgramState): StaticProgramState { - const files: StaticProgramState['files'] = {}; - for (const file of this.program.getSourceFiles()) - files[file.fileName] = { - ...oldState && oldState.files[file.fileName], + private isFileUpToDate(fileName: string, index: number, oldState: StaticProgramState) { + const fileNameQueue = [fileName]; + const stateQueue = [index]; + const childCounts = []; + const circularDependenciesQueue: number[] = []; + const cycles: string[][] = []; + while (true) { + fileName = fileNameQueue[fileNameQueue.length - 1]; + switch (this.dependenciesUpToDate.get(fileName)) { + case false: + return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + case undefined: { + let earliestCircularDependency = Number.MAX_SAFE_INTEGER; + let childCount = 0; + + processDeps: { + for (const cycle of cycles) { + if (cycle.includes(fileName)) { + // we already know this is a circular dependency, don't continue with this one and simply mark the parent as circular + earliestCircularDependency = findCircularDependencyOfCycle(fileNameQueue, childCounts, circularDependenciesQueue, cycle); + break processDeps; + } + } + const old = oldState.files[stateQueue[stateQueue.length - 1]]; + const dependencies = this.resolver.getDependencies(fileName); + const keys = Object.keys(old.dependencies); + + if (dependencies.size !== keys.length) + return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + for (const key of keys) { + let newDeps = dependencies.get(key); + if (newDeps === undefined) + return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); // external references have changed + const oldDeps = old.dependencies[key]; + if (oldDeps === null) { + if (newDeps !== null) + return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + continue; + } + if (newDeps === null) + return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + newDeps = Array.from(new Set(newDeps)); + if (newDeps.length !== oldDeps.length) + return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + const newDepsWithHash = this.sortByHash(newDeps); + for (let i = 0; i < newDepsWithHash.length; ++i) { + const oldDepState = oldState.files[oldDeps[i]]; + if (newDepsWithHash[i].hash !== oldDepState.hash) + return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + if (!this.assumeChangesOnlyAffectDirectDependencies) { + const indexInQueue = findParent(stateQueue, childCounts, oldDeps[i]); + if (indexInQueue === -1) { + // no circular dependency + fileNameQueue.push(newDepsWithHash[i].fileName); + stateQueue.push(oldDeps[i]); + ++childCount; + } else if (indexInQueue < earliestCircularDependency && newDepsWithHash[i].fileName !== fileName){ + earliestCircularDependency = indexInQueue; + } + } + } + } + } + + if (earliestCircularDependency === Number.MAX_SAFE_INTEGER) { + this.dependenciesUpToDate.set(fileName, true); + } else { + const parentCircularDep = circularDependenciesQueue[circularDependenciesQueue.length - 1]; + if (parentCircularDep === Number.MAX_SAFE_INTEGER) { + cycles.push([fileName]); + } else { + cycles[cycles.length - 1].push(fileName); + } + if (earliestCircularDependency < parentCircularDep) + circularDependenciesQueue[circularDependenciesQueue.length - 1] = earliestCircularDependency; + } + if (childCount !== 0) { + childCounts.push(childCount); + circularDependenciesQueue.push(earliestCircularDependency); + continue; + } + } + } + + fileNameQueue.pop(); + stateQueue.pop(); + if (fileNameQueue.length === 0) + return true; + + while (--childCounts[childCounts.length - 1] === 0) { + childCounts.pop(); + stateQueue.pop(); + fileName = fileNameQueue.pop()!; + const earliestCircularDependency = circularDependenciesQueue.pop()!; + if (earliestCircularDependency >= stateQueue.length) { + this.dependenciesUpToDate.set(fileName, true); // cycle ends here + if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) + for (const f of cycles.pop()!) + this.dependenciesUpToDate.set(f, true); // update result for all files that had a circular dependency on this one + } else { + const parentCircularDep = circularDependenciesQueue[circularDependenciesQueue.length - 1]; + if (parentCircularDep === Number.MAX_SAFE_INTEGER) { + cycles[cycles.length - 1].push(fileName); // parent had no cycle, keep the existing one + } else if (!cycles[cycles.length - 1].includes(fileName)) { + cycles[cycles.length - 2].push(fileName, ...cycles.pop()!); // merge cycles + } + if (earliestCircularDependency < circularDependenciesQueue[circularDependenciesQueue.length - 1]) + circularDependenciesQueue[circularDependenciesQueue.length - 1] = earliestCircularDependency; + } + if (fileNameQueue.length === 0) + return true; + } + } + } + + private markSelfAndParentsAsOutdated(fileNameQueue: readonly string[], childCounts: readonly number[]) { + this.dependenciesUpToDate.set(fileNameQueue[0], false); + for (let i = 0, current = 0; i < childCounts.length; ++i) { + current += childCounts[i]; + this.dependenciesUpToDate.set(fileNameQueue[current], false); + } + return false; + } + + public save() { + this.statePersistence.saveState(this.project, this.aggregate()); + } + + private aggregate(): StaticProgramState { + const oldState = this.tryReuseOldState(); + const lookup: Record = {}; + const mapToIndex = ({fileName}: {fileName: string}) => lookup[this.relativePathNames.get(fileName)!]; + const mapDependencies = (dependencies: ReadonlyMap) => { + const result: Record = {}; + for (const [key, f] of dependencies) + result[key] = f === null + ? null + : this.sortByHash(Array.from(new Set(f))).map(mapToIndex); + return result; + }; + const files: StaticProgramState.FileState[] = []; + const sourceFiles = this.program.getSourceFiles(); + for (let i = 0; i < sourceFiles.length; ++i) + lookup[this.getRelativePath(sourceFiles[i].fileName)] = i; + for (const file of sourceFiles) { + const relativePath = this.relativePathNames.get(file.fileName)!; + // TODO need to check each file individually for up to date + files.push({ + ...oldState && relativePath in oldState.lookup && oldState.files[oldState.lookup[relativePath]], hash: this.getFileHash(file.fileName), - dependencies: this.resolver.getDependencies(file.fileName), + dependencies: mapDependencies(this.resolver.getDependencies(file.fileName)), ...this.fileResults.get(file.fileName), - }; + }); + } return { + ts: ts.version, files, - optionsHash: this.optionsHash, - filesAffectingGlobalScope: this.resolver.getFilesAffectingGlobalScope(), + lookup, + global: this.sortByHash(this.resolver.getFilesAffectingGlobalScope()).map(mapToIndex), + options: this.optionsHash, }; } + + private sortByHash(fileNames: readonly string[]) { + return fileNames + .map((f) => ({fileName: f, hash: this.getFileHash(f)})) + .sort(compareHashKey); + } +} + +function findCircularDependencyOfCycle(fileNameQueue: readonly string[], childCounts: readonly number[], circularDependencies: readonly number[], cycle: readonly string[]) { + for (let i = 0, current = 0; i < childCounts.length; ++i) { + current += childCounts[i]; + const dep = circularDependencies[current]; + if (dep !== Number.MAX_SAFE_INTEGER && cycle.includes(fileNameQueue[current])) + return dep; + } + throw new Error('should never happen'); +} + +function findParent(stateQueue: readonly number[], childCounts: readonly number[], needle: number): number { + if (stateQueue[0] === needle) + return 0; + for (let i = 0, current = 0; i < childCounts.length; ++i) { + current += childCounts[i]; + if (stateQueue[current] === needle) + return current; + } + return -1; +} + +function compareHashKey(a: {hash: string}, b: {hash: string}) { + return a.hash < b.hash ? -1 : a.hash === b.hash ? 0 : 1; } function computeCompilerOptionsHash(options: ts.CompilerOptions) { const obj: Record = {}; for (const key of Object.keys(options).sort()) if (isKnownCompilerOption(key)) - obj[key] = options[key]; + obj[key] = options[key]; // TODO make paths relative and use correct casing return '' + djb2(JSON.stringify(obj)); } @@ -120,33 +338,41 @@ function isKnownCompilerOption(option: string): boolean { {[K in keyof ts.CompilerOptions]: string extends K ? never : K} extends {[K in keyof ts.CompilerOptions]: infer P} ? P : never; const o = option; switch (o) { - case 'allowJs': // + case 'allowJs': case 'allowSyntheticDefaultImports': + case 'allowUmdGlobalAccess': case 'allowUnreachableCode': case 'allowUnusedLabels': case 'alwaysStrict': + case 'assumeChangesOnlyAffectDirectDependencies': case 'baseUrl': case 'charset': case 'checkJs': case 'composite': case 'declaration': - case 'declarationDir': // - case 'declarationMap': // - case 'disableSizeLimit': // + case 'declarationDir': + case 'declarationMap': + case 'disableReferencedProjectLoad': + case 'disableSizeLimit': + case 'disableSourceOfProjectReferenceRedirect': + case 'disableSolutionSearching': case 'downlevelIteration': - case 'emitBOM': // - case 'emitDeclarationOnly': // + case 'emitBOM': + case 'emitDeclarationOnly': case 'emitDecoratorMetadata': case 'esModuleInterop': case 'experimentalDecorators': case 'forceConsistentCasingInFileNames': case 'importHelpers': - case 'incremental': // - case 'inlineSourceMap': // - case 'inlineSources': // + case 'importsNotUsedAsValues': + case 'incremental': + case 'inlineSourceMap': + case 'inlineSources': case 'isolatedModules': case 'jsx': case 'jsxFactory': + case 'jsxFragmentFactory': + case 'jsxImportSource': case 'keyofStringsOnly': case 'lib': case 'locale': @@ -154,10 +380,10 @@ function isKnownCompilerOption(option: string): boolean { case 'maxNodeModuleJsDepth': case 'module': case 'moduleResolution': - case 'newLine': // - case 'noEmit': // - case 'noEmitHelpers': // - case 'noEmitOnError': // + case 'newLine': + case 'noEmit': + case 'noEmitHelpers': + case 'noEmitOnError': case 'noErrorTruncation': case 'noFallthroughCasesInSwitch': case 'noImplicitAny': @@ -165,26 +391,28 @@ function isKnownCompilerOption(option: string): boolean { case 'noImplicitThis': case 'noImplicitUseStrict': case 'noLib': + case 'noPropertyAccessFromIndexSignature': case 'noResolve': case 'noStrictGenericChecks': + case 'noUncheckedIndexedAccess': case 'noUnusedLocals': case 'noUnusedParameters': - case 'out': // - case 'outDir': // - case 'outFile': // + case 'out': + case 'outDir': + case 'outFile': case 'paths': - case 'preserveConstEnums': // + case 'preserveConstEnums': case 'preserveSymlinks': - case 'project': // + case 'project': case 'reactNamespace': - case 'removeComments': // + case 'removeComments': case 'resolveJsonModule': case 'rootDir': case 'rootDirs': case 'skipDefaultLibCheck': case 'skipLibCheck': - case 'sourceMap': // - case 'sourceRoot': // + case 'sourceMap': + case 'sourceRoot': case 'strict': case 'strictBindCallApply': case 'strictFunctionTypes': @@ -194,13 +422,37 @@ function isKnownCompilerOption(option: string): boolean { case 'suppressExcessPropertyErrors': case 'suppressImplicitAnyIndexErrors': case 'target': - case 'traceResolution': // - case 'tsBuildInfoFile': // + case 'traceResolution': + case 'tsBuildInfoFile': case 'typeRoots': case 'types': + case 'useDefineForClassFields': return true; default: type AssertNever = T; return >false; } } + +// TODO this should probably happen in runner +function stripConfig(config: ReducedConfiguration) { + return { + rules: mapToObject(config.rules, stripRule), + settings: mapToObject(config.settings, identity), + }; +} + +function mapToObject(map: ReadonlyMap, transform: (v: T) => U) { + const result: Record = {}; + for (const [key, value] of map) + result[key] = transform(value); + return result; +} + +function identity(v: T) { + return v; +} + +function stripRule({rulesDirectories, ...rest}: EffectiveConfiguration.RuleConfig) { + return rest; +} diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index 97b664d2a..f408f0975 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -23,7 +23,6 @@ import { ProjectHost } from './project-host'; import debug = require('debug'); import { normalizeGlob } from 'normalize-glob'; import { ProgramStateFactory } from './program-state'; -import { StatePersistence } from './state-persistence'; const log = debug('wotan:runner'); @@ -59,7 +58,6 @@ export class Runner { private logger: MessageHandler, private filterFactory: FileFilterFactory, private programStateFactory: ProgramStateFactory, - private statePersistence: StatePersistence, ) {} public lintCollection(options: LintOptions): LintResult { @@ -93,10 +91,8 @@ export class Runner { for (let {files, program, configFilePath: tsconfigPath} of this.getFilesAndProgram(options.project, options.files, options.exclude, processorHost, options.references) ) { - const oldState = options.cache ? this.statePersistence.loadState(tsconfigPath) : undefined; - const programState = options.cache ? this.programStateFactory.create(program, processorHost) : undefined; + const programState = options.cache ? this.programStateFactory.create(program, processorHost, tsconfigPath) : undefined; let invalidatedProgram = false; - let updatedFiles: string[] = []; const factory: ProgramFactory = { getCompilerOptions() { return program.getCompilerOptions(); @@ -118,16 +114,12 @@ export class Runner { const effectiveConfig = config && this.configManager.reduce(config, originalName); if (effectiveConfig === undefined) continue; - if (programState !== undefined && updatedFiles.length !== 0) { - // TODO this is not correct as Program still contains the old SourceFile - programState.update(program, updatedFiles); - updatedFiles = []; - } let sourceFile = program.getSourceFile(file)!; const originalContent = mapped === undefined ? sourceFile.text : mapped.originalContent; let summary: FileSummary; const fix = shouldFix(sourceFile, options, originalName); if (fix) { + let updatedFile = false; summary = this.linter.lintAndFix( sourceFile, originalContent, @@ -140,7 +132,7 @@ export class Runner { log("Autofixing caused syntax errors in '%s', rolling back", sourceFile.fileName); sourceFile = ts.updateSourceFile(sourceFile, oldContent, invertChangeRange(range)); } else { - updatedFiles.push(sourceFile.fileName); + updatedFile = true; } // either way we need to store the new SourceFile as the old one is now corrupted processorHost.updateSourceFile(sourceFile); @@ -148,31 +140,35 @@ export class Runner { }, fix === true ? undefined : fix, factory, - mapped === undefined ? undefined : mapped.processor, + mapped?.processor, linterOptions, // pass cached results so we can apply fixes from cache - programState && programState.getUpToDateResult(file, effectiveConfig, oldState), + programState?.getUpToDateResult(file, effectiveConfig), ); + if (updatedFile) + programState?.update(factory.getProgram(), sourceFile.fileName); } else { summary = { - findings: programState && programState.getUpToDateResult(file, effectiveConfig, oldState) || + findings: programState?.getUpToDateResult(file, effectiveConfig) || this.linter.getFindings( sourceFile, effectiveConfig, factory, - mapped === undefined ? undefined : mapped.processor, + mapped?.processor, linterOptions, ), fixes: 0, content: originalContent, }; } - if (programState !== undefined) - programState.setFileResult(file, effectiveConfig, summary.findings); + programState?.setFileResult(file, effectiveConfig, summary.findings); yield [originalName, summary]; } - if (programState !== undefined) - this.statePersistence.saveState(tsconfigPath, programState.aggregate(oldState)); + // TODO always use a ProgramState when fixing, but only load old state if --cache is enabled + // loop over the files for n iterations while fixes are applied + // after each fix, create a new old state from the previous old state and the current state + // that way we know whether we need to check the first file again if the last file was changed via autofix + programState?.save(); } } diff --git a/packages/wotan/src/state-persistence.ts b/packages/wotan/src/state-persistence.ts index d4460ae4a..acc87cee3 100644 --- a/packages/wotan/src/state-persistence.ts +++ b/packages/wotan/src/state-persistence.ts @@ -1,45 +1,52 @@ import { injectable } from 'inversify'; import { CachedFileSystem } from './services/cached-file-system'; import { StaticProgramState } from './program-state'; -import { safeLoad, safeDump } from 'js-yaml'; import debug = require('debug'); -import { DirectoryService } from '@fimbul/ymir'; +import * as yaml from 'js-yaml'; const log = debug('wotan:statePersistence'); interface CacheFileContent { - version: string; - projects: Record; + v: string; + state: StaticProgramState; } const CACHE_VERSION = '1'; @injectable() export class StatePersistence { - constructor(private fs: CachedFileSystem, private dir: DirectoryService) {} + constructor(private fs: CachedFileSystem) {} public loadState(project: string): StaticProgramState | undefined { - const content = this.loadExisting(); - return content && content.projects[project]; // TODO resolve all paths + const fileName = buildFilename(project); + if (!this.fs.isFile(fileName)) + return; + try { + log("Loading cache from '%s'", fileName); + const content = yaml.load(this.fs.readFile(fileName)); + if (content?.v !== CACHE_VERSION) { + log("Version mismatch: expected '%s', actual: '%s'", CACHE_VERSION, content?.v); + return; + } + return content.state; + } catch { + log("Error loading cache '%s'", fileName); + return; + } } public saveState(project: string, state: StaticProgramState) { - const content = this.loadExisting() || {version: CACHE_VERSION, projects: {}}; - content.projects[project] = state; // TODO make all paths relative - this.fs.writeFile(this.dir.getCurrentDirectory() + '/.fimbullintercache.yaml', safeDump(content, {indent: 2, sortKeys: true})); - } - - private loadExisting(): CacheFileContent | undefined { - const fileName = this.dir.getCurrentDirectory() + '/.fimbullintercache.yaml'; - if (!this.fs.isFile(fileName)) { - log('%s does not exist', fileName); - return; - } - const content = safeLoad(this.fs.readFile(fileName))!; - if (content.version !== CACHE_VERSION) { - log('cache version mismatch'); - return; + const fileName = buildFilename(project); + log("Writing cache '%s'", fileName); + try { + const content: CacheFileContent = {v: CACHE_VERSION, state}; + this.fs.writeFile(fileName, yaml.dump(content, {indent: 2, sortKeys: true})); + } catch { + log("Error writing cache '%s'", fileName); } - return content; } } + +function buildFilename(tsconfigPath: string) { + return tsconfigPath.replace(/.[^.]+$/, '.fimbullintercache'); +} diff --git a/packages/wotan/src/utils.ts b/packages/wotan/src/utils.ts index 89eb25b73..564b6d870 100644 --- a/packages/wotan/src/utils.ts +++ b/packages/wotan/src/utils.ts @@ -217,12 +217,3 @@ export function djb2(str: string) { hash = ((hash << 5) + hash) + str.charCodeAt(i); return hash; } - -export function arraysAreEqual(a: ReadonlyArray, b: ReadonlyArray) { - if (a.length !== b.length) - return false; - for (let i = 0; i < a.length; ++i) - if (a[i] !== b[i]) - return false; - return true; -} diff --git a/packages/ymir/package.json b/packages/ymir/package.json index edeeece59..aac7b3c2e 100644 --- a/packages/ymir/package.json +++ b/packages/ymir/package.json @@ -26,7 +26,8 @@ "dependencies": { "inversify": "^5.0.0", "reflect-metadata": "^0.1.12", - "tslib": "^2.0.0" + "tslib": "^2.0.0", + "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" }, "devDependencies": { "tsutils": "^3.5.0" diff --git a/yarn.lock b/yarn.lock index 116db2a04..12cb73bfe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3773,6 +3773,12 @@ tslint@^5.0.0: tslib "^1.8.0" tsutils "^2.29.0" +tsutils@../../../tsutils/tsutils-3.19.1.tgz, tsutils@^3.19.1, tsutils@^3.5.0, tsutils@^3.5.1: + version "3.19.1" + resolved "../../../tsutils/tsutils-3.19.1.tgz#51c10066443cd4fd8818700f3b231ecbc522e0f1" + dependencies: + tslib "^1.8.1" + tsutils@^2.29.0: version "2.29.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-2.29.0.tgz#32b488501467acbedd4b85498673a0812aca0b99" @@ -3780,13 +3786,6 @@ tsutils@^2.29.0: dependencies: tslib "^1.8.1" -tsutils@^3.18.0, tsutils@^3.19.1, tsutils@^3.5.0, tsutils@^3.5.1: - version "3.19.1" - resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.19.1.tgz#d8566e0c51c82f32f9c25a4d367cd62409a547a9" - integrity sha512-GEdoBf5XI324lu7ycad7s6laADfnAqCw6wLGI+knxvw9vsIYBaJfYdmeCEG3FMMUiSm3OGgNb+m6utsWf5h9Vw== - dependencies: - tslib "^1.8.1" - ttypescript@^1.5.5: version "1.5.12" resolved "https://registry.yarnpkg.com/ttypescript/-/ttypescript-1.5.12.tgz#27a8356d7d4e719d0075a8feb4df14b52384f044" From 6609d2bfe1f24799f109c62f105798e34e83ddaa Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Wed, 27 Jan 2021 21:36:18 +0100 Subject: [PATCH 03/37] cleanup --- .fimbullinter.yaml | 1 - packages/bifrost/package.json | 2 +- packages/disir/package.json | 2 +- packages/heimdall/package.json | 3 +- packages/mithotyn/package.json | 3 +- packages/valtyr/package.json | 2 +- packages/ve/package.json | 1 - packages/wotan/package.json | 2 +- packages/wotan/src/di/core.module.ts | 6 +- packages/wotan/src/di/default.module.ts | 4 + packages/wotan/src/runner.ts | 17 +- .../default}/state-persistence.ts | 6 +- .../src/{ => services}/dependency-resolver.ts | 23 +- .../wotan/src/{ => services}/program-state.ts | 202 +++++++++++------- packages/ymir/package.json | 2 +- packages/ymir/src/index.ts | 39 ++++ tslint.json | 2 +- yarn.lock | 176 ++++++++------- 18 files changed, 289 insertions(+), 204 deletions(-) rename packages/wotan/src/{ => services/default}/state-persistence.ts (88%) rename packages/wotan/src/{ => services}/dependency-resolver.ts (93%) rename packages/wotan/src/{ => services}/program-state.ts (68%) diff --git a/.fimbullinter.yaml b/.fimbullinter.yaml index b66b1fc53..4ddb30663 100644 --- a/.fimbullinter.yaml +++ b/.fimbullinter.yaml @@ -1,4 +1,3 @@ project: - tsconfig.json reportUselessDirectives: true -cache: true diff --git a/packages/bifrost/package.json b/packages/bifrost/package.json index 796e5af12..41c6bbf89 100644 --- a/packages/bifrost/package.json +++ b/packages/bifrost/package.json @@ -33,7 +33,7 @@ "@fimbul/ymir": "^0.22.0", "get-caller-file": "^2.0.0", "tslib": "^2.0.0", - "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" + "tsutils": "^3.5.0" }, "devDependencies": { "@types/get-caller-file": "^1.0.0" diff --git a/packages/disir/package.json b/packages/disir/package.json index 4b7984e67..2be4cea72 100644 --- a/packages/disir/package.json +++ b/packages/disir/package.json @@ -7,7 +7,7 @@ "dependencies": { "@fimbul/ymir": "^0.22.0", "tslib": "^2.0.0", - "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" + "tsutils": "^3.5.0" }, "peerDependencies": { "typescript": ">= 4.0.0 || >= 4.2.0-dev" diff --git a/packages/heimdall/package.json b/packages/heimdall/package.json index c05451660..f9c9d7e09 100644 --- a/packages/heimdall/package.json +++ b/packages/heimdall/package.json @@ -33,7 +33,6 @@ "@fimbul/bifrost": "^0.22.0", "inversify": "^5.0.0", "tslib": "^2.0.0", - "tslint": "^5.0.0", - "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" + "tslint": "^5.0.0" } } diff --git a/packages/mithotyn/package.json b/packages/mithotyn/package.json index 99aced50f..58764f78a 100644 --- a/packages/mithotyn/package.json +++ b/packages/mithotyn/package.json @@ -32,8 +32,7 @@ "editor" ], "dependencies": { - "mock-require": "^3.0.2", - "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" + "mock-require": "^3.0.2" }, "devDependencies": { "@fimbul/wotan": "^0.16.0", diff --git a/packages/valtyr/package.json b/packages/valtyr/package.json index d46e58637..0f33c8533 100644 --- a/packages/valtyr/package.json +++ b/packages/valtyr/package.json @@ -39,6 +39,6 @@ "inversify": "^5.0.0", "tslib": "^2.0.0", "tslint": "^5.0.0", - "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" + "tsutils": "^3.5.0" } } diff --git a/packages/ve/package.json b/packages/ve/package.json index 23533c5f2..6eb97fe1c 100644 --- a/packages/ve/package.json +++ b/packages/ve/package.json @@ -32,7 +32,6 @@ "@fimbul/ymir": "^0.22.0", "parse5-sax-parser": "^6.0.0", "tslib": "^2.0.0", - "tsutils": "../../../tsutils/tsutils-3.19.1.tgz", "void-elements": "^3.1.0" }, "devDependencies": { diff --git a/packages/wotan/package.json b/packages/wotan/package.json index 518715ed1..dcd728741 100644 --- a/packages/wotan/package.json +++ b/packages/wotan/package.json @@ -59,7 +59,7 @@ "semver": "^7.0.0", "stable": "^0.1.8", "tslib": "^2.0.0", - "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" + "tsutils": "^3.20.0" }, "peerDependencies": { "typescript": ">= 4.0.0 || >= 4.2.0-dev" diff --git a/packages/wotan/src/di/core.module.ts b/packages/wotan/src/di/core.module.ts index aa4183e2e..7e677a106 100644 --- a/packages/wotan/src/di/core.module.ts +++ b/packages/wotan/src/di/core.module.ts @@ -7,9 +7,8 @@ import { Linter } from '../linter'; import { Runner } from '../runner'; import { ProcessorLoader } from '../services/processor-loader'; import { GlobalOptions } from '@fimbul/ymir'; -import { ProgramStateFactory } from '../program-state'; -import { StatePersistence } from '../state-persistence'; -import { DependencyResolverFactory } from '../dependency-resolver'; +import { ProgramStateFactory } from '../services/program-state'; +import { DependencyResolverFactory } from '../services/dependency-resolver'; export function createCoreModule(globalOptions: GlobalOptions) { return new ContainerModule((bind) => { @@ -21,7 +20,6 @@ export function createCoreModule(globalOptions: GlobalOptions) { bind(Linter).toSelf(); bind(Runner).toSelf(); bind(ProgramStateFactory).toSelf(); - bind(StatePersistence).toSelf(); // TODO allow overriding bind(DependencyResolverFactory).toSelf(); bind(GlobalOptions).toConstantValue(globalOptions); }); diff --git a/packages/wotan/src/di/default.module.ts b/packages/wotan/src/di/default.module.ts index e75930baa..d2360ff02 100644 --- a/packages/wotan/src/di/default.module.ts +++ b/packages/wotan/src/di/default.module.ts @@ -13,6 +13,7 @@ import { LineSwitchParser, BuiltinResolver, FileFilterFactory, + StatePersistence, } from '@fimbul/ymir'; import { NodeFormatterLoader } from '../services/default/formatter-loader-host'; import { NodeRuleLoader } from '../services/default/rule-loader-host'; @@ -26,6 +27,7 @@ import { DefaultConfigurationProvider } from '../services/default/configuration- import { DefaultLineSwitchParser, LineSwitchFilterFactory } from '../services/default/line-switches'; import { DefaultBuiltinResolver } from '../services/default/builtin-resolver'; import { DefaultFileFilterFactory } from '../services/default/file-filter'; +import { DefaultStatePersistence } from '../services/default/state-persistence'; export function createDefaultModule() { return new ContainerModule((bind, _unbind, isBound) => { @@ -55,5 +57,7 @@ export function createDefaultModule() { bind(BuiltinResolver).to(DefaultBuiltinResolver); if (!isBound(FileFilterFactory)) bind(FileFilterFactory).to(DefaultFileFilterFactory); + if (!isBound(StatePersistence)) + bind(StatePersistence).to(DefaultStatePersistence); }); } diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index f408f0975..c48fb40ef 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -22,7 +22,7 @@ import { ConfigurationManager } from './services/configuration-manager'; import { ProjectHost } from './project-host'; import debug = require('debug'); import { normalizeGlob } from 'normalize-glob'; -import { ProgramStateFactory } from './program-state'; +import { ProgramStateFactory } from './services/program-state'; const log = debug('wotan:runner'); @@ -118,6 +118,8 @@ export class Runner { const originalContent = mapped === undefined ? sourceFile.text : mapped.originalContent; let summary: FileSummary; const fix = shouldFix(sourceFile, options, originalName); + // TODO consider reportUselessDirectives in cache + const resultFromCache = programState?.getUpToDateResult(sourceFile.fileName, effectiveConfig); if (fix) { let updatedFile = false; summary = this.linter.lintAndFix( @@ -125,6 +127,7 @@ export class Runner { originalContent, effectiveConfig, (content, range) => { + invalidatedProgram = true; const oldContent = sourceFile.text; sourceFile = ts.updateSourceFile(sourceFile, content, range); const hasErrors = hasParseErrors(sourceFile); @@ -143,14 +146,13 @@ export class Runner { mapped?.processor, linterOptions, // pass cached results so we can apply fixes from cache - programState?.getUpToDateResult(file, effectiveConfig), + resultFromCache, ); if (updatedFile) programState?.update(factory.getProgram(), sourceFile.fileName); } else { summary = { - findings: programState?.getUpToDateResult(file, effectiveConfig) || - this.linter.getFindings( + findings: resultFromCache ?? this.linter.getFindings( sourceFile, effectiveConfig, factory, @@ -161,13 +163,10 @@ export class Runner { content: originalContent, }; } - programState?.setFileResult(file, effectiveConfig, summary.findings); + if (programState !== undefined && resultFromCache !== summary.findings) + programState.setFileResult(file, effectiveConfig, summary.findings); yield [originalName, summary]; } - // TODO always use a ProgramState when fixing, but only load old state if --cache is enabled - // loop over the files for n iterations while fixes are applied - // after each fix, create a new old state from the previous old state and the current state - // that way we know whether we need to check the first file again if the last file was changed via autofix programState?.save(); } } diff --git a/packages/wotan/src/state-persistence.ts b/packages/wotan/src/services/default/state-persistence.ts similarity index 88% rename from packages/wotan/src/state-persistence.ts rename to packages/wotan/src/services/default/state-persistence.ts index acc87cee3..b670aa8b4 100644 --- a/packages/wotan/src/state-persistence.ts +++ b/packages/wotan/src/services/default/state-persistence.ts @@ -1,8 +1,8 @@ import { injectable } from 'inversify'; -import { CachedFileSystem } from './services/cached-file-system'; -import { StaticProgramState } from './program-state'; +import { CachedFileSystem } from '../cached-file-system'; import debug = require('debug'); import * as yaml from 'js-yaml'; +import { StatePersistence, StaticProgramState } from '@fimbul/ymir'; const log = debug('wotan:statePersistence'); @@ -14,7 +14,7 @@ interface CacheFileContent { const CACHE_VERSION = '1'; @injectable() -export class StatePersistence { +export class DefaultStatePersistence implements StatePersistence { constructor(private fs: CachedFileSystem) {} public loadState(project: string): StaticProgramState | undefined { diff --git a/packages/wotan/src/dependency-resolver.ts b/packages/wotan/src/services/dependency-resolver.ts similarity index 93% rename from packages/wotan/src/dependency-resolver.ts rename to packages/wotan/src/services/dependency-resolver.ts index 59a60728a..57cc16de0 100644 --- a/packages/wotan/src/dependency-resolver.ts +++ b/packages/wotan/src/services/dependency-resolver.ts @@ -1,9 +1,9 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; import { isModuleDeclaration, isNamespaceExportDeclaration, findImports, ImportKind, isCompilerOptionEnabled } from 'tsutils'; -import { resolveCachedResult, getOutputFileNamesOfProjectReference, iterateProjectReferences } from './utils'; +import { resolveCachedResult, getOutputFileNamesOfProjectReference, iterateProjectReferences } from '../utils'; import bind from 'bind-decorator'; -import { ProjectHost } from './project-host'; +import { ProjectHost } from '../project-host'; export interface DependencyResolver { update(program: ts.Program, updatedFile: string): void; @@ -18,7 +18,7 @@ export class DependencyResolverFactory { } } -export interface DependencyResolverState { +interface DependencyResolverState { affectsGlobalScope: readonly string[]; ambientModules: ReadonlyMap; moduleAugmentations: ReadonlyMap; @@ -72,8 +72,8 @@ class DependencyResolverImpl implements DependencyResolver { for (const file of files) { const resolved = this.getExternalReferences(file).get(module); // if an augmentation's identifier can be resolved from the declaring file, the augmentation applies to the resolved path - if (resolved != null) { - addToList(moduleAugmentations, resolved, file); + if (resolved !== null) { + addToList(moduleAugmentations, resolved!, file); } else { // if a pattern ambient module matches the augmented identifier, the augmentation applies to that const matchingPattern = getBestMatchingPattern(module, patternAmbientModules.keys()); @@ -95,7 +95,7 @@ class DependencyResolverImpl implements DependencyResolver { return (this.state ??= this.buildState()).affectsGlobalScope; } - public getDependencies(file: string) { // TODO is it worth caching this? + public getDependencies(file: string) { this.state ??= this.buildState(); const result = new Map(); for (const [identifier, resolved] of this.getExternalReferences(file)) { @@ -120,7 +120,10 @@ class DependencyResolverImpl implements DependencyResolver { const meta = this.fileMetadata.get(file)!; if (!meta.isExternalModule) for (const ambientModule of meta.ambientModules) - result.set(ambientModule, this.state[ambientModule.includes('*') ? 'patternAmbientModules' : 'ambientModules'].get(ambientModule)!); + result.set( + ambientModule, + this.state[ambientModule.includes('*') ? 'patternAmbientModules' : 'ambientModules'].get(ambientModule)!, + ); return result; } @@ -142,14 +145,12 @@ class DependencyResolverImpl implements DependencyResolver { private collectExternalReferences(fileName: string): Map { // TODO useSourceOfProjectReferenceRedirect // TODO referenced files + // TODO add tslib if importHelpers is enabled const sourceFile = this.program.getSourceFile(fileName)!; const references = new Set(findImports(sourceFile, ImportKind.All, false).map(({text}) => text)); - if (ts.isExternalModule(sourceFile)) { - // if (isCompilerOptionEnabled(this.compilerOptions, 'importHelpers')) - // references.add('tslib') + if (ts.isExternalModule(sourceFile)) for (const augmentation of this.getFileMetaData(fileName).ambientModules) references.add(augmentation); - } const result = new Map(); if (references.size === 0) return result; diff --git a/packages/wotan/src/program-state.ts b/packages/wotan/src/services/program-state.ts similarity index 68% rename from packages/wotan/src/program-state.ts rename to packages/wotan/src/services/program-state.ts index 75358e494..41431c759 100644 --- a/packages/wotan/src/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -1,38 +1,21 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; import { DependencyResolver, DependencyResolverFactory } from './dependency-resolver'; -import { resolveCachedResult, djb2 } from './utils'; +import { resolveCachedResult, djb2 } from '../utils'; import bind from 'bind-decorator'; -import { EffectiveConfiguration, Finding, ReducedConfiguration } from '@fimbul/ymir'; +import { EffectiveConfiguration, Finding, ReducedConfiguration, StatePersistence, StaticProgramState } from '@fimbul/ymir'; import debug = require('debug'); -import { ProjectHost } from './project-host'; +import { ProjectHost } from '../project-host'; import { isCompilerOptionEnabled } from 'tsutils'; -import { StatePersistence } from './state-persistence'; import * as path from 'path'; const log = debug('wotan:programState'); -export interface StaticProgramState { - // TODO add linter version - /** TypeScript version */ - readonly ts: string; - /** Hash of compilerOptions */ - readonly options: string; - /** Maps filename to index in 'files' array */ - readonly lookup: Readonly>; - /** Index of files that affect global scope */ - readonly global: readonly number[]; - /** Information about all files in the program */ - readonly files: readonly StaticProgramState.FileState[]; -} - -export namespace StaticProgramState { - export interface FileState { - readonly hash: string; - readonly dependencies: Readonly>; - readonly result?: readonly Finding[]; - readonly config?: string; - } +export interface ProgramState { + update(program: ts.Program, updatedFile: string): void; + getUpToDateResult(fileName: string, config: EffectiveConfiguration): readonly Finding[] | undefined; + setFileResult(fileName: string, config: EffectiveConfiguration, result: readonly Finding[]): void; + save(): void; } @injectable() @@ -40,48 +23,62 @@ export class ProgramStateFactory { constructor(private resolverFactory: DependencyResolverFactory, private statePersistence: StatePersistence) {} public create(program: ts.Program, host: ProjectHost, tsconfigPath: string) { - return new ProgramState(program, this.resolverFactory.create(host, program), this.statePersistence, tsconfigPath); + return new ProgramStateImpl(program, this.resolverFactory.create(host, program), this.statePersistence, tsconfigPath); } } -class ProgramState { +interface FileResults { + readonly config: string; + readonly result: ReadonlyArray; +} + +const oldStateSymbol = Symbol('oldState'); +class ProgramStateImpl implements ProgramState { private optionsHash = computeCompilerOptionsHash(this.program.getCompilerOptions()); private assumeChangesOnlyAffectDirectDependencies = isCompilerOptionEnabled(this.program.getCompilerOptions(), 'assumeChangesOnlyAffectDirectDependencies'); private fileHashes = new Map(); - private fileResults = new Map}>(); + private fileResults = new Map(); private relativePathNames = new Map(); - private _oldState: StaticProgramState | undefined; + private [oldStateSymbol]: StaticProgramState | undefined; private recheckOldState = true; private projectDirectory = path.posix.dirname(this.project); private dependenciesUpToDate = new Map(); - constructor(private program: ts.Program, private resolver: DependencyResolver, private statePersistence: StatePersistence, private project: string) { + constructor( + private program: ts.Program, + private resolver: DependencyResolver, + private statePersistence: StatePersistence, + private project: string, + ) { const oldState = this.statePersistence.loadState(project); - this._oldState = (oldState?.ts !== ts.version || oldState.options !== this.optionsHash) ? undefined : oldState; + this[oldStateSymbol] = (oldState?.ts !== ts.version || oldState.options !== this.optionsHash) ? undefined : oldState; } /** get old state if global files didn't change */ private tryReuseOldState() { - if (this._oldState === undefined || !this.recheckOldState) - return this._oldState; + const oldState = this[oldStateSymbol]; + if (oldState === undefined || !this.recheckOldState) + return oldState; const filesAffectingGlobalScope = this.resolver.getFilesAffectingGlobalScope(); - if (this._oldState.global.length !== filesAffectingGlobalScope.length) - return this._oldState = undefined; + if (oldState.global.length !== filesAffectingGlobalScope.length) + return this[oldStateSymbol] = undefined; const globalFilesWithHash = this.sortByHash(filesAffectingGlobalScope); for (let i = 0; i < globalFilesWithHash.length; ++i) { - const index = this._oldState.global[i]; + const index = oldState.global[i]; if ( - globalFilesWithHash[i].hash !== this._oldState.files[index].hash || - !this.assumeChangesOnlyAffectDirectDependencies && !this.isFileUpToDate(globalFilesWithHash[i].fileName, index, this._oldState) + globalFilesWithHash[i].hash !== oldState.files[index].hash || + !this.assumeChangesOnlyAffectDirectDependencies && + !this.fileDependenciesUpToDate(globalFilesWithHash[i].fileName, index, oldState) ) - return this._oldState = undefined; + return this[oldStateSymbol] = undefined; } this.recheckOldState = false; - return this._oldState; + return oldState; } public update(program: ts.Program, updatedFile: string) { + this.program = program; this.resolver.update(program, updatedFile); this.fileHashes.delete(updatedFile); this.recheckOldState = true; @@ -119,7 +116,7 @@ class ProgramState { old.result === undefined || old.config !== '' + djb2(JSON.stringify(stripConfig(config))) || old.hash !== this.getFileHash(fileName) || - !this.isFileUpToDate(fileName, index, oldState) + !this.fileDependenciesUpToDate(fileName, index, oldState) ) return; log('reusing state for %s', fileName); @@ -127,58 +124,81 @@ class ProgramState { } public setFileResult(fileName: string, config: ReducedConfiguration, result: ReadonlyArray) { + if (!this.isFileUpToDate(fileName)) { + log('File %s is outdated, merging current state into old state', fileName); + // we need to create a state where the file is up-to-date + // so we replace the old state with the current state + // this includes all results from old state that were still up-to-date and all file results if they were still valid + this[oldStateSymbol] = this.aggregate(); + this.recheckOldState = false; + this.fileResults = new Map(); + this.dependenciesUpToDate = new Map(this.program.getSourceFiles().map((f) => [f.fileName, true])); + } this.fileResults.set(fileName, {result, config: '' + djb2(JSON.stringify(stripConfig(config)))}); } - private isFileUpToDate(fileName: string, index: number, oldState: StaticProgramState) { + private isFileUpToDate(fileName: string): boolean { + const oldState = this.tryReuseOldState(); + if (oldState === undefined) + return false; + const relative = this.getRelativePath(fileName); + if (!(relative in oldState.lookup)) + return false; + const index = oldState.lookup[relative]; + return oldState.files[index].hash === this.getFileHash(fileName) && + this.fileDependenciesUpToDate(fileName, index, oldState); + } + + private fileDependenciesUpToDate(fileName: string, index: number, oldState: StaticProgramState): boolean { const fileNameQueue = [fileName]; const stateQueue = [index]; const childCounts = []; const circularDependenciesQueue: number[] = []; - const cycles: string[][] = []; + const cycles: Array> = []; while (true) { fileName = fileNameQueue[fileNameQueue.length - 1]; switch (this.dependenciesUpToDate.get(fileName)) { case false: - return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); case undefined: { let earliestCircularDependency = Number.MAX_SAFE_INTEGER; let childCount = 0; - processDeps: { - for (const cycle of cycles) { - if (cycle.includes(fileName)) { - // we already know this is a circular dependency, don't continue with this one and simply mark the parent as circular - earliestCircularDependency = findCircularDependencyOfCycle(fileNameQueue, childCounts, circularDependenciesQueue, cycle); - break processDeps; - } + for (const cycle of cycles) { + if (cycle.has(fileName)) { + // we already know this is a circular dependency, skip this one and simply mark the parent as circular + earliestCircularDependency = + findCircularDependencyOfCycle(fileNameQueue, childCounts, circularDependenciesQueue, cycle); + break; } + } + if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) { const old = oldState.files[stateQueue[stateQueue.length - 1]]; const dependencies = this.resolver.getDependencies(fileName); const keys = Object.keys(old.dependencies); if (dependencies.size !== keys.length) - return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); for (const key of keys) { let newDeps = dependencies.get(key); if (newDeps === undefined) - return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); // external references have changed + return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); const oldDeps = old.dependencies[key]; if (oldDeps === null) { if (newDeps !== null) - return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); continue; } if (newDeps === null) - return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); newDeps = Array.from(new Set(newDeps)); if (newDeps.length !== oldDeps.length) - return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); const newDepsWithHash = this.sortByHash(newDeps); for (let i = 0; i < newDepsWithHash.length; ++i) { const oldDepState = oldState.files[oldDeps[i]]; if (newDepsWithHash[i].hash !== oldDepState.hash) - return this.markSelfAndParentsAsOutdated(fileNameQueue, childCounts); + return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); if (!this.assumeChangesOnlyAffectDirectDependencies) { const indexInQueue = findParent(stateQueue, childCounts, oldDeps[i]); if (indexInQueue === -1) { @@ -186,7 +206,7 @@ class ProgramState { fileNameQueue.push(newDepsWithHash[i].fileName); stateQueue.push(oldDeps[i]); ++childCount; - } else if (indexInQueue < earliestCircularDependency && newDepsWithHash[i].fileName !== fileName){ + } else if (indexInQueue < earliestCircularDependency && newDepsWithHash[i].fileName !== fileName) { earliestCircularDependency = indexInQueue; } } @@ -199,9 +219,9 @@ class ProgramState { } else { const parentCircularDep = circularDependenciesQueue[circularDependenciesQueue.length - 1]; if (parentCircularDep === Number.MAX_SAFE_INTEGER) { - cycles.push([fileName]); + cycles.push(new Set([fileName])); } else { - cycles[cycles.length - 1].push(fileName); + cycles[cycles.length - 1].add(fileName); } if (earliestCircularDependency < parentCircularDep) circularDependenciesQueue[circularDependenciesQueue.length - 1] = earliestCircularDependency; @@ -228,13 +248,17 @@ class ProgramState { this.dependenciesUpToDate.set(fileName, true); // cycle ends here if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) for (const f of cycles.pop()!) - this.dependenciesUpToDate.set(f, true); // update result for all files that had a circular dependency on this one + this.dependenciesUpToDate.set(f, true); // update result for files that had a circular dependency on this one } else { const parentCircularDep = circularDependenciesQueue[circularDependenciesQueue.length - 1]; if (parentCircularDep === Number.MAX_SAFE_INTEGER) { - cycles[cycles.length - 1].push(fileName); // parent had no cycle, keep the existing one - } else if (!cycles[cycles.length - 1].includes(fileName)) { - cycles[cycles.length - 2].push(fileName, ...cycles.pop()!); // merge cycles + cycles[cycles.length - 1].add(fileName); // parent had no cycle, keep the existing one + } else if (!cycles[cycles.length - 1].has(fileName)) { + const currentCycle = cycles.pop()!; + const previousCycle = cycles[cycles.length - 1]; + previousCycle.add(fileName); + for (const f of currentCycle) + previousCycle.add(f); // merge cycles } if (earliestCircularDependency < circularDependenciesQueue[circularDependenciesQueue.length - 1]) circularDependenciesQueue[circularDependenciesQueue.length - 1] = earliestCircularDependency; @@ -245,15 +269,6 @@ class ProgramState { } } - private markSelfAndParentsAsOutdated(fileNameQueue: readonly string[], childCounts: readonly number[]) { - this.dependenciesUpToDate.set(fileNameQueue[0], false); - for (let i = 0, current = 0; i < childCounts.length; ++i) { - current += childCounts[i]; - this.dependenciesUpToDate.set(fileNameQueue[current], false); - } - return false; - } - public save() { this.statePersistence.saveState(this.project, this.aggregate()); } @@ -276,18 +291,26 @@ class ProgramState { lookup[this.getRelativePath(sourceFiles[i].fileName)] = i; for (const file of sourceFiles) { const relativePath = this.relativePathNames.get(file.fileName)!; - // TODO need to check each file individually for up to date + let results = this.fileResults.get(file.fileName); + if (results === undefined && oldState !== undefined && relativePath in oldState.lookup) { + const old = oldState.files[oldState.lookup[relativePath]]; + if (old.result !== undefined) + results = old; + } + if (results !== undefined && !this.isFileUpToDate(file.fileName)) { + log('Discarding outdated results for %s', file.fileName); + results = undefined; + } files.push({ - ...oldState && relativePath in oldState.lookup && oldState.files[oldState.lookup[relativePath]], + ...results, hash: this.getFileHash(file.fileName), dependencies: mapDependencies(this.resolver.getDependencies(file.fileName)), - ...this.fileResults.get(file.fileName), }); } return { - ts: ts.version, files, lookup, + ts: ts.version, global: this.sortByHash(this.resolver.getFilesAffectingGlobalScope()).map(mapToIndex), options: this.optionsHash, }; @@ -300,11 +323,18 @@ class ProgramState { } } -function findCircularDependencyOfCycle(fileNameQueue: readonly string[], childCounts: readonly number[], circularDependencies: readonly number[], cycle: readonly string[]) { +function findCircularDependencyOfCycle( + fileNameQueue: readonly string[], + childCounts: readonly number[], + circularDependencies: readonly number[], + cycle: ReadonlySet, +) { + if (circularDependencies[0] !== Number.MAX_SAFE_INTEGER && cycle.has(fileNameQueue[0])) + return circularDependencies[0]; for (let i = 0, current = 0; i < childCounts.length; ++i) { current += childCounts[i]; const dep = circularDependencies[current]; - if (dep !== Number.MAX_SAFE_INTEGER && cycle.includes(fileNameQueue[current])) + if (dep !== Number.MAX_SAFE_INTEGER && cycle.has(fileNameQueue[current])) return dep; } throw new Error('should never happen'); @@ -321,6 +351,15 @@ function findParent(stateQueue: readonly number[], childCounts: readonly number[ return -1; } +function markSelfAndParentsAsOutdated(fileNameQueue: readonly string[], childCounts: readonly number[], results: Map) { + results.set(fileNameQueue[0], false); + for (let i = 0, current = 0; i < childCounts.length; ++i) { + current += childCounts[i]; + results.set(fileNameQueue[current], false); + } + return false; +} + function compareHashKey(a: {hash: string}, b: {hash: string}) { return a.hash < b.hash ? -1 : a.hash === b.hash ? 0 : 1; } @@ -334,7 +373,7 @@ function computeCompilerOptionsHash(options: ts.CompilerOptions) { } function isKnownCompilerOption(option: string): boolean { - type KnownOptions = + type KnownOptions = // tslint:disable-next-line:no-unused {[K in keyof ts.CompilerOptions]: string extends K ? never : K} extends {[K in keyof ts.CompilerOptions]: infer P} ? P : never; const o = option; switch (o) { @@ -434,7 +473,6 @@ function isKnownCompilerOption(option: string): boolean { } } -// TODO this should probably happen in runner function stripConfig(config: ReducedConfiguration) { return { rules: mapToObject(config.rules, stripRule), @@ -453,6 +491,6 @@ function identity(v: T) { return v; } -function stripRule({rulesDirectories, ...rest}: EffectiveConfiguration.RuleConfig) { +function stripRule({rulesDirectories: _ignored, ...rest}: EffectiveConfiguration.RuleConfig) { return rest; } diff --git a/packages/ymir/package.json b/packages/ymir/package.json index aac7b3c2e..4dfa8660b 100644 --- a/packages/ymir/package.json +++ b/packages/ymir/package.json @@ -27,7 +27,7 @@ "inversify": "^5.0.0", "reflect-metadata": "^0.1.12", "tslib": "^2.0.0", - "tsutils": "../../../tsutils/tsutils-3.19.1.tgz" + "tsutils": "^3.20.0" }, "devDependencies": { "tsutils": "^3.5.0" diff --git a/packages/ymir/src/index.ts b/packages/ymir/src/index.ts index e97f13a5f..2a5eef7b3 100644 --- a/packages/ymir/src/index.ts +++ b/packages/ymir/src/index.ts @@ -515,3 +515,42 @@ export interface FileFilter { /** @returns `true` if the file should be linted, false if it should be filtered out. Intended for use in `Array.prototype.filter`. */ filter(file: ts.SourceFile): boolean; } + +export interface StatePersistence { + loadState(project: string): StaticProgramState | undefined; + saveState(project: string, state: StaticProgramState): void; +} +export abstract class StatePersistence {} + +export interface StaticProgramState { + /** TypeScript version */ + readonly ts: string; + /** Hash of compilerOptions */ + readonly options: string; + /** Maps filename to index in 'files' array */ + readonly lookup: Readonly>; + /** Index of files that affect global scope */ + readonly global: readonly number[]; + /** Information about all files in the program */ + readonly files: readonly StaticProgramState.FileState[]; +} + +export namespace StaticProgramState { + export interface FileState { + /** Hash of file contents */ + readonly hash: string; + /** + * Key: module specifier as referenced in the file, order may be random + * Value: - `null` if dependency could not be resolved + * - List of files (or rather their index) that the module specifier resolves to. + * That is the actual file at that path and/or files containing `declare module "..."` for that module specifier. + * May contain the current file. + * This list is ordered by the hash of the files ascending, + */ + readonly dependencies: Readonly>; + /** The list of findings if this file has up-to-date results */ + readonly result?: readonly Finding[]; + /** Hash of the configuration used to produce `result` for this file */ + readonly config?: string; + } +} diff --git a/tslint.json b/tslint.json index 30fcc81f1..b6d0021d1 100644 --- a/tslint.json +++ b/tslint.json @@ -54,7 +54,7 @@ "ignore-params" ], "no-irregular-whitespace": true, - "no-null-keyword": true, + "no-null-keyword": false, "no-return-await": true, "no-shadowed-variable": true, "no-sparse-arrays": true, diff --git a/yarn.lock b/yarn.lock index 12cb73bfe..5b91affdd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -322,29 +322,29 @@ "@octokit/types" "^6.0.0" universal-user-agent "^6.0.0" -"@octokit/openapi-types@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-2.2.0.tgz#123e0438a0bc718ccdac3b5a2e69b3dd00daa85b" - integrity sha512-274lNUDonw10kT8wHg8fCcUc1ZjZHbWv0/TbAwb0ojhBQqZYc1cQ/4yqTVTtPMDeZ//g7xVEYe/s3vURkRghPg== +"@octokit/openapi-types@^3.2.0": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-3.2.0.tgz#d62d0ff7147dbf4d218616b2484ee2a5d023055d" + integrity sha512-X7yW/fpzF3uTAE+LbPD3HEeeU+/49o0V4kNA/yv8jQ3BDpFayv/osTOhY1y1mLXljW2bOJcOCSGZo4jFKPJ6Vw== "@octokit/plugin-paginate-rest@^2.6.2": - version "2.7.0" - resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.7.0.tgz#6bb7b043c246e0654119a6ec4e72a172c9e2c7f3" - integrity sha512-+zARyncLjt9b0FjqPAbJo4ss7HOlBi1nprq+cPlw5vu2+qjy7WvlXhtXFdRHQbSL1Pt+bfAKaLADEkkvg8sP8w== + version "2.8.0" + resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.8.0.tgz#2b41e12b494e895bf5fb5b12565d2c80a0ecc6ae" + integrity sha512-HtuEQ2AYE4YFEBQN0iHmMsIvVucd5RsnwJmRKIsfAg1/ZeoMaU+jXMnTAZqIUEmcVJA27LjHUm3f1hxf8Fpdxw== dependencies: - "@octokit/types" "^6.0.1" + "@octokit/types" "^6.4.0" "@octokit/plugin-request-log@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@octokit/plugin-request-log/-/plugin-request-log-1.0.2.tgz#394d59ec734cd2f122431fbaf05099861ece3c44" integrity sha512-oTJSNAmBqyDR41uSMunLQKMX0jmEXbwD1fpz8FG27lScV3RhtGfBa1/BBLym+PxcC16IBlF7KH9vP1BUYxA+Eg== -"@octokit/plugin-rest-endpoint-methods@4.4.1": - version "4.4.1" - resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-4.4.1.tgz#105cf93255432155de078c9efc33bd4e14d1cd63" - integrity sha512-+v5PcvrUcDeFXf8hv1gnNvNLdm4C0+2EiuWt9EatjjUmfriM1pTMM+r4j1lLHxeBQ9bVDmbywb11e3KjuavieA== +"@octokit/plugin-rest-endpoint-methods@4.8.0": + version "4.8.0" + resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-4.8.0.tgz#c1f24f940fc265f0021c8f544e3d8755f3253759" + integrity sha512-2zRpXDveJH8HsXkeeMtRW21do8wuSxVn1xXFdvhILyxlLWqGQrdJUA1/dk5DM7iAAYvwT/P3bDOLs90yL4S2AA== dependencies: - "@octokit/types" "^6.1.0" + "@octokit/types" "^6.5.0" deprecation "^2.3.1" "@octokit/request-error@^2.0.0": @@ -371,21 +371,21 @@ universal-user-agent "^6.0.0" "@octokit/rest@^18.0.0": - version "18.0.12" - resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.0.12.tgz#278bd41358c56d87c201e787e8adc0cac132503a" - integrity sha512-hNRCZfKPpeaIjOVuNJzkEL6zacfZlBPV8vw8ReNeyUkVvbuCvvrrx8K8Gw2eyHHsmd4dPlAxIXIZ9oHhJfkJpw== + version "18.0.14" + resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.0.14.tgz#a152478465746542e80697b5a5576ccb6151dc4d" + integrity sha512-62mKIaBb/XD2Z2KCBmAPydEk/d0IBMOnwk6DJVo36ICTnxlRPTdQwFE2LzlpBPDR52xOKPlGqb3Bnhh99atltA== dependencies: "@octokit/core" "^3.2.3" "@octokit/plugin-paginate-rest" "^2.6.2" "@octokit/plugin-request-log" "^1.0.2" - "@octokit/plugin-rest-endpoint-methods" "4.4.1" + "@octokit/plugin-rest-endpoint-methods" "4.8.0" -"@octokit/types@^6.0.0", "@octokit/types@^6.0.1", "@octokit/types@^6.0.3", "@octokit/types@^6.1.0": - version "6.2.1" - resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.2.1.tgz#7f881fe44475ab1825776a4a59ca1ae082ed1043" - integrity sha512-jHs9OECOiZxuEzxMZcXmqrEO8GYraHF+UzNVH2ACYh8e/Y7YoT+hUf9ldvVd6zIvWv4p3NdxbQ0xx3ku5BnSiA== +"@octokit/types@^6.0.0", "@octokit/types@^6.0.3", "@octokit/types@^6.4.0", "@octokit/types@^6.5.0": + version "6.5.0" + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.5.0.tgz#8f27c52d57eb4096fb05a290f4afc90194e08b19" + integrity sha512-mzCy7lkYQv+kM58W37uTg/mWoJ4nvRDRCkjSdqlrgA28hJEYNJTMYiGTvmq39cdtnMPJd0hshysBEAaH4D5C7w== dependencies: - "@octokit/openapi-types" "^2.2.0" + "@octokit/openapi-types" "^3.2.0" "@types/node" ">= 8" "@sindresorhus/is@^0.14.0": @@ -485,14 +485,14 @@ "@types/node" "*" "@types/node@*", "@types/node@>= 8": - version "14.14.20" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.20.tgz#f7974863edd21d1f8a494a73e8e2b3658615c340" - integrity sha512-Y93R97Ouif9JEOWPIUyU+eyIdyRqQR0I8Ez1dzku4hDx34NWh4HbtIc3WNzwB1Y9ULvNGeu5B8h8bVL5cAk4/A== + version "14.14.22" + resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.22.tgz#0d29f382472c4ccf3bd96ff0ce47daf5b7b84b18" + integrity sha512-g+f/qj/cNcqKkc3tFqlXOYjrmZA+jNBiDzbP3kH+B+otKFqAdPgVTGP1IeKRdMml/aE69as5S4FqtxAbl+LaMw== "@types/node@^10.12.0": - version "10.17.50" - resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.50.tgz#7a20902af591282aa9176baefc37d4372131c32d" - integrity sha512-vwX+/ija9xKc/z9VqMCdbf4WYcMTGsI0I/L/6shIF3qXURxZOhPQlPRHtjTpiNhAwn0paMJzlOQqw6mAGEQnTA== + version "10.17.51" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.51.tgz#639538575befbcf3d3861f95c41de8e47124d674" + integrity sha512-KANw+MkL626tq90l++hGelbl67irOJzGhUJk6a1Bt8QHOeh9tztJx+L0AqttraWKinmZn7Qi5lJZJzx45Gq0dg== "@types/normalize-package-data@^2.4.0": version "2.4.0" @@ -648,9 +648,9 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: color-convert "^2.0.1" ansi-styles@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.0.0.tgz#675dbbb5ca1908fa90abe4e5b1c2e9b1f4080d99" - integrity sha512-6564t0m0fuQMnockqBv7wJxo9T5C2V9JpYXyNScfRDPVLusOQQhkpMGrFC17QbiolraQ1sMXX+Y5nJpjqozL4g== + version "5.1.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.1.0.tgz#a436bcc51d81f4fab5080b2ba94242e377c41573" + integrity sha512-osxifZo3ar56+e8tdYreU6p8FZGciBHo5O0JoDAxMUqZuyNUb+yHEwYtJZ+Z32R459jEgtwVf1u8D7qYwU0l6w== anymatch@~3.1.1: version "3.1.1" @@ -924,10 +924,10 @@ caching-transform@^4.0.0: package-hash "^4.0.0" write-file-atomic "^3.0.0" -call-bind@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.1.tgz#29aca9151f8ddcfd5b9b786898f005f425e88567" - integrity sha512-tvAvUwNcRikl3RVF20X9lsYmmepsovzTWeJiXjO0PkJp15uy/6xKFZOQtuiSULwYW+6ToZBprphCgWXC2dSgcQ== +call-bind@^1.0.0, call-bind@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== dependencies: function-bind "^1.1.1" get-intrinsic "^1.0.2" @@ -978,9 +978,9 @@ chardet@^0.7.0: integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== chokidar@^3.4.3: - version "3.5.0" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.0.tgz#458a4816a415e9d3b3caa4faec2b96a6935a9e65" - integrity sha512-JgQM9JS92ZbFR4P90EvmzNpSGhpPBGBSj10PILeDyYFwp4h2/D9OM03wsJ4zW1fEp4ka2DGrnUeD7FuvQ2aZ2Q== + version "3.5.1" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a" + integrity sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw== dependencies: anymatch "~3.1.1" braces "~3.0.2" @@ -1126,10 +1126,10 @@ combined-stream@^1.0.6, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" -commander@6.2.1: - version "6.2.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" - integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== +commander@7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-7.0.0.tgz#3e2bbfd8bb6724760980988fb5b22b7ee6b71ab2" + integrity sha512-ovx/7NkTrnPuIV8sqk/GjUIIM1+iUQeqA3ye2VNpq9sVoiZsooObWlQy+OPWGI17GDaEoybuAGJm6U8yC077BA== commander@^2.12.1: version "2.20.3" @@ -1310,9 +1310,9 @@ delayed-stream@~1.0.0: integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= dependency-cruiser@^9.0.0: - version "9.21.6" - resolved "https://registry.yarnpkg.com/dependency-cruiser/-/dependency-cruiser-9.21.6.tgz#6634353872aa8a6cb375054eb717b0b81d2a7fb6" - integrity sha512-HFHCFQnqWgcQi4+oJRt/m9NUoitaMRTV5TQ+XnRZcj/+TA2mS6ROSIaBY0MfTzA2VF11dguc1ViCq6M49CSSaA== + version "9.22.0" + resolved "https://registry.yarnpkg.com/dependency-cruiser/-/dependency-cruiser-9.22.0.tgz#4467bc37331c93fc41258e67732d45871e88df0b" + integrity sha512-XlPQmo+fM4jeKVXYLf9bvMKiP0vs9pTSyvMs2HFibBHMDm22XaOWtJ2hsRV9QDk2RSptQshkYm2JY0x7A3yJww== dependencies: acorn "8.0.4" acorn-jsx "5.3.1" @@ -1321,7 +1321,7 @@ dependency-cruiser@^9.0.0: acorn-walk "8.0.1" ajv "7.0.3" chalk "4.1.0" - commander "6.2.1" + commander "7.0.0" enhanced-resolve "5.1.0" figures "3.2.0" get-stream "6.0.0" @@ -1416,9 +1416,9 @@ enhanced-resolve@5.1.0: tapable "^2.0.0" enhanced-resolve@^4.0.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz#3b806f3bfafc1ec7de69551ef93cca46c1704126" - integrity sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ== + version "4.5.0" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.5.0.tgz#2f3cfd84dbe3b487f18f2db2ef1e064a571ca5ec" + integrity sha512-Nv9m36S/vxpsI+Hc4/ZGRs0n9mXqSWGGq49zxb/cJfPAQMbUtttJAlNPS4AQzaBdw/pKskw5bMbekT/Y7W/Wlg== dependencies: graceful-fs "^4.1.2" memory-fs "^0.5.0" @@ -1444,22 +1444,24 @@ error-ex@^1.3.1: is-arrayish "^0.2.1" es-abstract@^1.18.0-next.1: - version "1.18.0-next.1" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0-next.1.tgz#6e3a0a4bda717e5023ab3b8e90bec36108d22c68" - integrity sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA== + version "1.18.0-next.2" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0-next.2.tgz#088101a55f0541f595e7e057199e27ddc8f3a5c2" + integrity sha512-Ih4ZMFHEtZupnUh6497zEL4y2+w8+1ljnCyaTa+adcoafI1GOvMwFlDjBLfWR7y9VLfrjRJe9ocuHY1PSR9jjw== dependencies: + call-bind "^1.0.2" es-to-primitive "^1.2.1" function-bind "^1.1.1" + get-intrinsic "^1.0.2" has "^1.0.3" has-symbols "^1.0.1" is-callable "^1.2.2" - is-negative-zero "^2.0.0" + is-negative-zero "^2.0.1" is-regex "^1.1.1" - object-inspect "^1.8.0" + object-inspect "^1.9.0" object-keys "^1.1.1" - object.assign "^4.1.1" - string.prototype.trimend "^1.0.1" - string.prototype.trimstart "^1.0.1" + object.assign "^4.1.2" + string.prototype.trimend "^1.0.3" + string.prototype.trimstart "^1.0.3" es-to-primitive@^1.2.1: version "1.2.1" @@ -1545,9 +1547,9 @@ fast-diff@^1.2.0: integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== fast-glob@^3.1.1: - version "3.2.4" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.4.tgz#d20aefbf99579383e7f3cc66529158c9b98554d3" - integrity sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ== + version "3.2.5" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.5.tgz#7939af2a656de79a4f1901903ee8adcaa7cb9661" + integrity sha512-2DtFcgT68wiTTiwZ2hNdJfcHNke9XOfnwmBRWXhmeKM8rF0TGwmC/Qto3S7RoZKp5cilZbxzO5iTNTQsJ+EeDg== dependencies: "@nodelib/fs.stat" "^2.0.2" "@nodelib/fs.walk" "^1.2.3" @@ -2074,7 +2076,7 @@ is-negated-glob@^1.0.0: resolved "https://registry.yarnpkg.com/is-negated-glob/-/is-negated-glob-1.0.0.tgz#6910bca5da8c95e784b5751b976cf5a10fee36d2" integrity sha1-aRC8pdqMleeEtXUbl2z1oQ/uNtI= -is-negative-zero@^2.0.0: +is-negative-zero@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" integrity sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w== @@ -2737,7 +2739,7 @@ oauth-sign@~0.9.0: resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== -object-inspect@^1.8.0: +object-inspect@^1.9.0: version "1.9.0" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.9.0.tgz#c90521d74e1127b67266ded3394ad6116986533a" integrity sha512-i3Bp9iTqwhaLZBxGkRfo5ZbE07BQRT7MGu8+nNgwW9ItGp1TzCTw2DLEoWwjClxBjOFI/hWljTAmYGCEwmtnOw== @@ -2747,7 +2749,7 @@ object-keys@^1.0.12, object-keys@^1.1.1: resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== -object.assign@^4.1.1: +object.assign@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940" integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ== @@ -2772,9 +2774,9 @@ onetime@^5.1.0: mimic-fn "^2.1.0" ora@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/ora/-/ora-5.2.0.tgz#de10bfd2d15514384af45f3fa9d9b1aaf344fda1" - integrity sha512-+wG2v8TUU8EgzPHun1k/n45pXquQ9fHnbXVetl9rRgO6kjZszGGbraF3XPTIdgeA+s1lbRjSEftAnyT0w8ZMvQ== + version "5.3.0" + resolved "https://registry.yarnpkg.com/ora/-/ora-5.3.0.tgz#fb832899d3a1372fe71c8b2c534bbfe74961bb6f" + integrity sha512-zAKMgGXUim0Jyd6CXK9lraBnD3H5yPGBPPOkC23a2BG6hsm4Zu6OQSjQuEtV0BHDf4aKHcUFvJiGRrFuW3MG8g== dependencies: bl "^4.0.3" chalk "^4.1.0" @@ -2888,9 +2890,9 @@ parse-json@^4.0.0: json-parse-better-errors "^1.0.1" parse-json@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.1.0.tgz#f96088cdf24a8faa9aea9a009f2d9d942c999646" - integrity sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ== + version "5.2.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" + integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== dependencies: "@babel/code-frame" "^7.0.0" error-ex "^1.3.1" @@ -3133,9 +3135,9 @@ reflect-metadata@^0.1.12: integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg== regexp-tree@~0.1.1: - version "0.1.21" - resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.21.tgz#55e2246b7f7d36f1b461490942fa780299c400d7" - integrity sha512-kUUXjX4AnqnR8KRTCrayAo9PzYMRKmVoGgaz2tBuz0MF3g1ZbGebmtW0yFHfFK9CmBjQKeYIgoL22pFLBJY7sw== + version "0.1.23" + resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.23.tgz#8a8ce1cc5e971acef62213a7ecdb1f6e18a1f1b2" + integrity sha512-+7HWfb4Bvu8Rs2eQTUIpX9I/PlQkYOuTNbRpKLJlQpSgwSkzFYh+pUj0gtvglnOZLKB6YgnIgRuJ2/IlpL48qw== registry-auth-token@^4.0.0: version "4.2.1" @@ -3520,7 +3522,7 @@ string.prototype.padend@^3.0.0: define-properties "^1.1.3" es-abstract "^1.18.0-next.1" -string.prototype.trimend@^1.0.1: +string.prototype.trimend@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.3.tgz#a22bd53cca5c7cf44d7c9d5c732118873d6cd18b" integrity sha512-ayH0pB+uf0U28CtjlLvL7NaohvR1amUvVZk+y3DYb0Ey2PUV5zPkkKy9+U1ndVEIXO8hNg18eIv9Jntbii+dKw== @@ -3528,7 +3530,7 @@ string.prototype.trimend@^1.0.1: call-bind "^1.0.0" define-properties "^1.1.3" -string.prototype.trimstart@^1.0.1: +string.prototype.trimstart@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.3.tgz#9b4cb590e123bb36564401d59824298de50fd5aa" integrity sha512-oBIBUy5lea5tt0ovtOFiEQaBkoBBkyJhZXzJYrSmDo5IUUqbOPvVezuRs/agBIdZ2p2Eo1FD6bD9USyBLfl3xg== @@ -3773,12 +3775,6 @@ tslint@^5.0.0: tslib "^1.8.0" tsutils "^2.29.0" -tsutils@../../../tsutils/tsutils-3.19.1.tgz, tsutils@^3.19.1, tsutils@^3.5.0, tsutils@^3.5.1: - version "3.19.1" - resolved "../../../tsutils/tsutils-3.19.1.tgz#51c10066443cd4fd8818700f3b231ecbc522e0f1" - dependencies: - tslib "^1.8.1" - tsutils@^2.29.0: version "2.29.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-2.29.0.tgz#32b488501467acbedd4b85498673a0812aca0b99" @@ -3786,6 +3782,20 @@ tsutils@^2.29.0: dependencies: tslib "^1.8.1" +tsutils@^3.19.1, tsutils@^3.5.0, tsutils@^3.5.1: + version "3.19.1" + resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.19.1.tgz#d8566e0c51c82f32f9c25a4d367cd62409a547a9" + integrity sha512-GEdoBf5XI324lu7ycad7s6laADfnAqCw6wLGI+knxvw9vsIYBaJfYdmeCEG3FMMUiSm3OGgNb+m6utsWf5h9Vw== + dependencies: + tslib "^1.8.1" + +tsutils@^3.20.0: + version "3.20.0" + resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.20.0.tgz#ea03ea45462e146b53d70ce0893de453ff24f698" + integrity sha512-RYbuQuvkhuqVeXweWT3tJLKOEJ/UUw9GjNEZGWdrLLlM+611o1gwLHBpxoFJKKl25fLprp2eVthtKs5JOrNeXg== + dependencies: + tslib "^1.8.1" + ttypescript@^1.5.5: version "1.5.12" resolved "https://registry.yarnpkg.com/ttypescript/-/ttypescript-1.5.12.tgz#27a8356d7d4e719d0075a8feb4df14b52384f044" @@ -3843,9 +3853,9 @@ typescript@4.2.0-dev.20210121: integrity sha512-3dq9PpGVO6lJOa1LtWVs6bsDpwvIqKhb4uC5YUWsBfXBBRpqxYfO2RKn/0v8WWgwD5dN7AeyIyEZIgRpPEekOQ== uglify-js@^3.1.4: - version "3.12.4" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.12.4.tgz#93de48bb76bb3ec0fc36563f871ba46e2ee5c7ee" - integrity sha512-L5i5jg/SHkEqzN18gQMTWsZk3KelRsfD1wUVNqtq0kzqWQqcJjyL8yc1o8hJgRrWqrAl2mUFbhfznEIoi7zi2A== + version "3.12.5" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.12.5.tgz#83241496087c640efe9dfc934832e71725aba008" + integrity sha512-SgpgScL4T7Hj/w/GexjnBHi3Ien9WS1Rpfg5y91WXMj9SY997ZCQU76mH4TpLwwfmMvoOU8wiaRkIf6NaH3mtg== underscore.string@~2.2.0rc: version "2.2.1" From 1ad610d0270b44fc479ba2297ab024d310a3e57b Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Wed, 27 Jan 2021 22:36:32 +0100 Subject: [PATCH 04/37] handle absolute paths in compilerOptions --- packages/wotan/src/services/program-state.ts | 233 ++++++++++--------- yarn.lock | 9 +- 2 files changed, 127 insertions(+), 115 deletions(-) diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 41431c759..2c108cf9e 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -23,7 +23,7 @@ export class ProgramStateFactory { constructor(private resolverFactory: DependencyResolverFactory, private statePersistence: StatePersistence) {} public create(program: ts.Program, host: ProjectHost, tsconfigPath: string) { - return new ProgramStateImpl(program, this.resolverFactory.create(host, program), this.statePersistence, tsconfigPath); + return new ProgramStateImpl(host, program, this.resolverFactory.create(host, program), this.statePersistence, tsconfigPath); } } @@ -34,7 +34,9 @@ interface FileResults { const oldStateSymbol = Symbol('oldState'); class ProgramStateImpl implements ProgramState { - private optionsHash = computeCompilerOptionsHash(this.program.getCompilerOptions()); + private projectDirectory = path.posix.dirname(this.project); + private canonicalProjectDirectory = this.host.getCanonicalFileName(this.projectDirectory); + private optionsHash = computeCompilerOptionsHash(this.program.getCompilerOptions(), this.projectDirectory); private assumeChangesOnlyAffectDirectDependencies = isCompilerOptionEnabled(this.program.getCompilerOptions(), 'assumeChangesOnlyAffectDirectDependencies'); private fileHashes = new Map(); @@ -42,10 +44,10 @@ class ProgramStateImpl implements ProgramState { private relativePathNames = new Map(); private [oldStateSymbol]: StaticProgramState | undefined; private recheckOldState = true; - private projectDirectory = path.posix.dirname(this.project); private dependenciesUpToDate = new Map(); constructor( + private host: ts.CompilerHost, private program: ts.Program, private resolver: DependencyResolver, private statePersistence: StatePersistence, @@ -100,7 +102,7 @@ class ProgramStateImpl implements ProgramState { @bind private makeRelativePath(fileName: string) { - return path.posix.relative(this.projectDirectory, fileName); + return path.posix.relative(this.canonicalProjectDirectory, this.host.getCanonicalFileName(fileName)); } public getUpToDateResult(fileName: string, config: ReducedConfiguration) { @@ -364,112 +366,129 @@ function compareHashKey(a: {hash: string}, b: {hash: string}) { return a.hash < b.hash ? -1 : a.hash === b.hash ? 0 : 1; } -function computeCompilerOptionsHash(options: ts.CompilerOptions) { +const enum CompilerOptionKind { + Ignore = 0, + Value = 1, + Path = 2, + PathArray = 3, +} + +type KnownCompilerOptions = // tslint:disable-next-line:no-unused + {[K in keyof ts.CompilerOptions]: string extends K ? never : K} extends {[K in keyof ts.CompilerOptions]: infer P} ? P : never; + +type AdditionalCompilerOptions = 'pathsBasePath'; + +const compilerOptionKinds: Record = { + allowJs: CompilerOptionKind.Value, + allowSyntheticDefaultImports: CompilerOptionKind.Value, + allowUmdGlobalAccess: CompilerOptionKind.Value, + allowUnreachableCode: CompilerOptionKind.Value, + allowUnusedLabels: CompilerOptionKind.Value, + alwaysStrict: CompilerOptionKind.Value, + assumeChangesOnlyAffectDirectDependencies: CompilerOptionKind.Value, + baseUrl: CompilerOptionKind.Path, + charset: CompilerOptionKind.Value, + checkJs: CompilerOptionKind.Value, + composite: CompilerOptionKind.Value, + declaration: CompilerOptionKind.Value, + declarationDir: CompilerOptionKind.Path, + declarationMap: CompilerOptionKind.Value, + disableReferencedProjectLoad: CompilerOptionKind.Ignore, + disableSizeLimit: CompilerOptionKind.Value, + disableSourceOfProjectReferenceRedirect: CompilerOptionKind.Value, + disableSolutionSearching: CompilerOptionKind.Ignore, + downlevelIteration: CompilerOptionKind.Value, + emitBOM: CompilerOptionKind.Value, + emitDeclarationOnly: CompilerOptionKind.Value, + emitDecoratorMetadata: CompilerOptionKind.Value, + esModuleInterop: CompilerOptionKind.Value, + experimentalDecorators: CompilerOptionKind.Value, + forceConsistentCasingInFileNames: CompilerOptionKind.Value, + importHelpers: CompilerOptionKind.Value, + importsNotUsedAsValues: CompilerOptionKind.Value, + incremental: CompilerOptionKind.Value, + inlineSourceMap: CompilerOptionKind.Value, + inlineSources: CompilerOptionKind.Value, + isolatedModules: CompilerOptionKind.Value, + jsx: CompilerOptionKind.Value, + jsxFactory: CompilerOptionKind.Value, + jsxFragmentFactory: CompilerOptionKind.Value, + jsxImportSource: CompilerOptionKind.Value, + keyofStringsOnly: CompilerOptionKind.Value, + lib: CompilerOptionKind.Value, + locale: CompilerOptionKind.Value, + mapRoot: CompilerOptionKind.Value, + maxNodeModuleJsDepth: CompilerOptionKind.Value, + module: CompilerOptionKind.Value, + moduleResolution: CompilerOptionKind.Value, + newLine: CompilerOptionKind.Value, + noEmit: CompilerOptionKind.Value, + noEmitHelpers: CompilerOptionKind.Value, + noEmitOnError: CompilerOptionKind.Value, + noErrorTruncation: CompilerOptionKind.Value, + noFallthroughCasesInSwitch: CompilerOptionKind.Value, + noImplicitAny: CompilerOptionKind.Value, + noImplicitReturns: CompilerOptionKind.Value, + noImplicitThis: CompilerOptionKind.Value, + noImplicitUseStrict: CompilerOptionKind.Value, + noLib: CompilerOptionKind.Value, + noPropertyAccessFromIndexSignature: CompilerOptionKind.Value, + noResolve: CompilerOptionKind.Value, + noStrictGenericChecks: CompilerOptionKind.Value, + noUncheckedIndexedAccess: CompilerOptionKind.Value, + noUnusedLocals: CompilerOptionKind.Value, + noUnusedParameters: CompilerOptionKind.Value, + out: CompilerOptionKind.Value, + outDir: CompilerOptionKind.Path, + outFile: CompilerOptionKind.Path, + paths: CompilerOptionKind.Value, + pathsBasePath: CompilerOptionKind.Path, + preserveConstEnums: CompilerOptionKind.Value, + preserveSymlinks: CompilerOptionKind.Value, + project: CompilerOptionKind.Ignore, + reactNamespace: CompilerOptionKind.Value, + removeComments: CompilerOptionKind.Value, + resolveJsonModule: CompilerOptionKind.Value, + rootDir: CompilerOptionKind.Path, + rootDirs: CompilerOptionKind.PathArray, + skipDefaultLibCheck: CompilerOptionKind.Value, + skipLibCheck: CompilerOptionKind.Value, + sourceMap: CompilerOptionKind.Value, + sourceRoot: CompilerOptionKind.Value, + strict: CompilerOptionKind.Value, + strictBindCallApply: CompilerOptionKind.Value, + strictFunctionTypes: CompilerOptionKind.Value, + strictNullChecks: CompilerOptionKind.Value, + strictPropertyInitialization: CompilerOptionKind.Value, + stripInternal: CompilerOptionKind.Value, + suppressExcessPropertyErrors: CompilerOptionKind.Value, + suppressImplicitAnyIndexErrors: CompilerOptionKind.Value, + target: CompilerOptionKind.Value, + traceResolution: CompilerOptionKind.Value, + tsBuildInfoFile: CompilerOptionKind.Ignore, + typeRoots: CompilerOptionKind.PathArray, + types: CompilerOptionKind.Value, + useDefineForClassFields: CompilerOptionKind.Value, +}; + +function computeCompilerOptionsHash(options: ts.CompilerOptions, relativeTo: string) { const obj: Record = {}; - for (const key of Object.keys(options).sort()) - if (isKnownCompilerOption(key)) - obj[key] = options[key]; // TODO make paths relative and use correct casing + for (const key of Object.keys(options).sort()) { + switch (compilerOptionKinds[key]) { + case CompilerOptionKind.Value: + obj[key] = options[key]; + break; + case CompilerOptionKind.Path: + obj[key] = makeRelativePath(options[key]); + break; + case CompilerOptionKind.PathArray: + obj[key] = (options[key]).map(makeRelativePath); + } + } return '' + djb2(JSON.stringify(obj)); -} -function isKnownCompilerOption(option: string): boolean { - type KnownOptions = // tslint:disable-next-line:no-unused - {[K in keyof ts.CompilerOptions]: string extends K ? never : K} extends {[K in keyof ts.CompilerOptions]: infer P} ? P : never; - const o = option; - switch (o) { - case 'allowJs': - case 'allowSyntheticDefaultImports': - case 'allowUmdGlobalAccess': - case 'allowUnreachableCode': - case 'allowUnusedLabels': - case 'alwaysStrict': - case 'assumeChangesOnlyAffectDirectDependencies': - case 'baseUrl': - case 'charset': - case 'checkJs': - case 'composite': - case 'declaration': - case 'declarationDir': - case 'declarationMap': - case 'disableReferencedProjectLoad': - case 'disableSizeLimit': - case 'disableSourceOfProjectReferenceRedirect': - case 'disableSolutionSearching': - case 'downlevelIteration': - case 'emitBOM': - case 'emitDeclarationOnly': - case 'emitDecoratorMetadata': - case 'esModuleInterop': - case 'experimentalDecorators': - case 'forceConsistentCasingInFileNames': - case 'importHelpers': - case 'importsNotUsedAsValues': - case 'incremental': - case 'inlineSourceMap': - case 'inlineSources': - case 'isolatedModules': - case 'jsx': - case 'jsxFactory': - case 'jsxFragmentFactory': - case 'jsxImportSource': - case 'keyofStringsOnly': - case 'lib': - case 'locale': - case 'mapRoot': - case 'maxNodeModuleJsDepth': - case 'module': - case 'moduleResolution': - case 'newLine': - case 'noEmit': - case 'noEmitHelpers': - case 'noEmitOnError': - case 'noErrorTruncation': - case 'noFallthroughCasesInSwitch': - case 'noImplicitAny': - case 'noImplicitReturns': - case 'noImplicitThis': - case 'noImplicitUseStrict': - case 'noLib': - case 'noPropertyAccessFromIndexSignature': - case 'noResolve': - case 'noStrictGenericChecks': - case 'noUncheckedIndexedAccess': - case 'noUnusedLocals': - case 'noUnusedParameters': - case 'out': - case 'outDir': - case 'outFile': - case 'paths': - case 'preserveConstEnums': - case 'preserveSymlinks': - case 'project': - case 'reactNamespace': - case 'removeComments': - case 'resolveJsonModule': - case 'rootDir': - case 'rootDirs': - case 'skipDefaultLibCheck': - case 'skipLibCheck': - case 'sourceMap': - case 'sourceRoot': - case 'strict': - case 'strictBindCallApply': - case 'strictFunctionTypes': - case 'strictNullChecks': - case 'strictPropertyInitialization': - case 'stripInternal': - case 'suppressExcessPropertyErrors': - case 'suppressImplicitAnyIndexErrors': - case 'target': - case 'traceResolution': - case 'tsBuildInfoFile': - case 'typeRoots': - case 'types': - case 'useDefineForClassFields': - return true; - default: - type AssertNever = T; - return >false; + function makeRelativePath(p: string) { + return path.posix.relative(relativeTo, p); } } diff --git a/yarn.lock b/yarn.lock index 5b91affdd..8f79daa2d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3782,14 +3782,7 @@ tsutils@^2.29.0: dependencies: tslib "^1.8.1" -tsutils@^3.19.1, tsutils@^3.5.0, tsutils@^3.5.1: - version "3.19.1" - resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.19.1.tgz#d8566e0c51c82f32f9c25a4d367cd62409a547a9" - integrity sha512-GEdoBf5XI324lu7ycad7s6laADfnAqCw6wLGI+knxvw9vsIYBaJfYdmeCEG3FMMUiSm3OGgNb+m6utsWf5h9Vw== - dependencies: - tslib "^1.8.1" - -tsutils@^3.20.0: +tsutils@^3.19.1, tsutils@^3.20.0, tsutils@^3.5.0, tsutils@^3.5.1: version "3.20.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.20.0.tgz#ea03ea45462e146b53d70ce0893de453ff24f698" integrity sha512-RYbuQuvkhuqVeXweWT3tJLKOEJ/UUw9GjNEZGWdrLLlM+611o1gwLHBpxoFJKKl25fLprp2eVthtKs5JOrNeXg== From 780bea53a66ef89785ad1440933568b8e02861e3 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 28 Jan 2021 19:43:58 +0100 Subject: [PATCH 05/37] major rewrite and fixing --- .../src/services/default/state-persistence.ts | 4 +- packages/wotan/src/services/program-state.ts | 211 ++++++++++-------- packages/ymir/src/index.ts | 2 + 3 files changed, 126 insertions(+), 91 deletions(-) diff --git a/packages/wotan/src/services/default/state-persistence.ts b/packages/wotan/src/services/default/state-persistence.ts index b670aa8b4..b41f5ef9f 100644 --- a/packages/wotan/src/services/default/state-persistence.ts +++ b/packages/wotan/src/services/default/state-persistence.ts @@ -7,11 +7,11 @@ import { StatePersistence, StaticProgramState } from '@fimbul/ymir'; const log = debug('wotan:statePersistence'); interface CacheFileContent { - v: string; + v: number; state: StaticProgramState; } -const CACHE_VERSION = '1'; +const CACHE_VERSION = 1; @injectable() export class DefaultStatePersistence implements StatePersistence { diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 2c108cf9e..293d48f8c 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -32,6 +32,14 @@ interface FileResults { readonly result: ReadonlyArray; } +const enum DependencyState { + Unknown = 0, + Outdated = 1, + Ok = 2, +} + +const STATE_VERSION = 1; + const oldStateSymbol = Symbol('oldState'); class ProgramStateImpl implements ProgramState { private projectDirectory = path.posix.dirname(this.project); @@ -44,7 +52,7 @@ class ProgramStateImpl implements ProgramState { private relativePathNames = new Map(); private [oldStateSymbol]: StaticProgramState | undefined; private recheckOldState = true; - private dependenciesUpToDate = new Map(); + private dependenciesUpToDate: Uint8Array; constructor( private host: ts.CompilerHost, @@ -54,7 +62,13 @@ class ProgramStateImpl implements ProgramState { private project: string, ) { const oldState = this.statePersistence.loadState(project); - this[oldStateSymbol] = (oldState?.ts !== ts.version || oldState.options !== this.optionsHash) ? undefined : oldState; + if (oldState?.v !== STATE_VERSION || oldState.ts !== ts.version || oldState.options !== this.optionsHash) { + this[oldStateSymbol] = undefined; + this.dependenciesUpToDate = new Uint8Array(0); + } else { + this[oldStateSymbol] = oldState; + this.dependenciesUpToDate = new Uint8Array(oldState.files.length); + } } /** get old state if global files didn't change */ @@ -84,7 +98,7 @@ class ProgramStateImpl implements ProgramState { this.resolver.update(program, updatedFile); this.fileHashes.delete(updatedFile); this.recheckOldState = true; - this.dependenciesUpToDate = new Map(); + this.dependenciesUpToDate.fill(DependencyState.Unknown); } private getFileHash(file: string) { @@ -131,10 +145,10 @@ class ProgramStateImpl implements ProgramState { // we need to create a state where the file is up-to-date // so we replace the old state with the current state // this includes all results from old state that were still up-to-date and all file results if they were still valid - this[oldStateSymbol] = this.aggregate(); + const newState = this[oldStateSymbol] = this.aggregate(); this.recheckOldState = false; this.fileResults = new Map(); - this.dependenciesUpToDate = new Map(this.program.getSourceFiles().map((f) => [f.fileName, true])); + this.dependenciesUpToDate = new Uint8Array(newState.files.length).fill(DependencyState.Ok); } this.fileResults.set(fileName, {result, config: '' + djb2(JSON.stringify(stripConfig(config)))}); } @@ -147,68 +161,86 @@ class ProgramStateImpl implements ProgramState { if (!(relative in oldState.lookup)) return false; const index = oldState.lookup[relative]; - return oldState.files[index].hash === this.getFileHash(fileName) && - this.fileDependenciesUpToDate(fileName, index, oldState); + if (oldState.files[index].hash !== this.getFileHash(fileName)) + return false; + switch (this.dependenciesUpToDate[index]) { + case DependencyState.Unknown: + return this.fileDependenciesUpToDate(fileName, index, oldState); + case DependencyState.Ok: + return true; + case DependencyState.Outdated: + return false; + } } - private fileDependenciesUpToDate(fileName: string, index: number, oldState: StaticProgramState): boolean { + private fileDependenciesUpToDate(fileName: string, index: number, oldState: StaticProgramState): boolean { // TODO something is wrong + // File names that are waiting to be processed, each iteration of the loop processes one file const fileNameQueue = [fileName]; - const stateQueue = [index]; + // For each entry in `fileNameQueue` this holds the index of that file in `oldState.files` + const indexQueue = [index]; + // If a file is waiting for its children to be processed, it is moved from `indexQueue` to `parents` + const parents: number[] = []; + // For each entry in `parents` this holds the number of children that still need to be processed for that file const childCounts = []; + // For each entry in `parents` this holds the index in of the earliest circular dependency in `parents`. + // For example, a value of `[Number.MAX_SAFE_INTEGER, 0]` means that `parents[1]` has a dependency on `parents[0]` (the root file). const circularDependenciesQueue: number[] = []; - const cycles: Array> = []; + // If a file has a circular on one of its parents, it is moved from `indexQueue` to the current cycle + // or creates a new cycle if its parent is not already in a cycle. + const cycles: Array> = []; while (true) { - fileName = fileNameQueue[fileNameQueue.length - 1]; - switch (this.dependenciesUpToDate.get(fileName)) { - case false: - return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); - case undefined: { + index = indexQueue.pop()!; + fileName = fileNameQueue.pop()!; + switch (this.dependenciesUpToDate[index]) { + case DependencyState.Outdated: + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + case DependencyState.Unknown: { let earliestCircularDependency = Number.MAX_SAFE_INTEGER; let childCount = 0; for (const cycle of cycles) { - if (cycle.has(fileName)) { + if (cycle.has(index)) { // we already know this is a circular dependency, skip this one and simply mark the parent as circular earliestCircularDependency = - findCircularDependencyOfCycle(fileNameQueue, childCounts, circularDependenciesQueue, cycle); + findCircularDependencyOfCycle(parents, circularDependenciesQueue, cycle); break; } } - if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) { - const old = oldState.files[stateQueue[stateQueue.length - 1]]; + if (earliestCircularDependency === Number.MAX_SAFE_INTEGER) { + const old = oldState.files[index]; const dependencies = this.resolver.getDependencies(fileName); const keys = Object.keys(old.dependencies); if (dependencies.size !== keys.length) - return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); for (const key of keys) { let newDeps = dependencies.get(key); if (newDeps === undefined) - return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); const oldDeps = old.dependencies[key]; if (oldDeps === null) { if (newDeps !== null) - return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); continue; } if (newDeps === null) - return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); newDeps = Array.from(new Set(newDeps)); if (newDeps.length !== oldDeps.length) - return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); const newDepsWithHash = this.sortByHash(newDeps); for (let i = 0; i < newDepsWithHash.length; ++i) { const oldDepState = oldState.files[oldDeps[i]]; if (newDepsWithHash[i].hash !== oldDepState.hash) - return markSelfAndParentsAsOutdated(fileNameQueue, childCounts, this.dependenciesUpToDate); - if (!this.assumeChangesOnlyAffectDirectDependencies) { - const indexInQueue = findParent(stateQueue, childCounts, oldDeps[i]); + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + if (!this.assumeChangesOnlyAffectDirectDependencies && fileName !== newDepsWithHash[i].fileName) { + const indexInQueue = parents.indexOf(oldDeps[i]) if (indexInQueue === -1) { // no circular dependency fileNameQueue.push(newDepsWithHash[i].fileName); - stateQueue.push(oldDeps[i]); + indexQueue.push(oldDeps[i]); ++childCount; - } else if (indexInQueue < earliestCircularDependency && newDepsWithHash[i].fileName !== fileName) { + } else if (indexInQueue < earliestCircularDependency) { earliestCircularDependency = indexInQueue; } } @@ -216,56 +248,36 @@ class ProgramStateImpl implements ProgramState { } } - if (earliestCircularDependency === Number.MAX_SAFE_INTEGER) { - this.dependenciesUpToDate.set(fileName, true); - } else { - const parentCircularDep = circularDependenciesQueue[circularDependenciesQueue.length - 1]; - if (parentCircularDep === Number.MAX_SAFE_INTEGER) { - cycles.push(new Set([fileName])); - } else { - cycles[cycles.length - 1].add(fileName); - } - if (earliestCircularDependency < parentCircularDep) - circularDependenciesQueue[circularDependenciesQueue.length - 1] = earliestCircularDependency; + if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) { + earliestCircularDependency = + setCircularDependency(parents, circularDependenciesQueue, index, cycles, earliestCircularDependency); + } else if (childCount === 0) { + this.dependenciesUpToDate[index] = DependencyState.Ok; } if (childCount !== 0) { + parents.push(index); childCounts.push(childCount); circularDependenciesQueue.push(earliestCircularDependency); continue; } } } + // we only get here for files with no children to process - fileNameQueue.pop(); - stateQueue.pop(); - if (fileNameQueue.length === 0) - return true; + if (parents.length === 0) + return true; // only happens if the initial file has no dependencies or they are all already known as Ok while (--childCounts[childCounts.length - 1] === 0) { + index = parents.pop()!; childCounts.pop(); - stateQueue.pop(); - fileName = fileNameQueue.pop()!; const earliestCircularDependency = circularDependenciesQueue.pop()!; - if (earliestCircularDependency >= stateQueue.length) { - this.dependenciesUpToDate.set(fileName, true); // cycle ends here + if (earliestCircularDependency >= parents.length) { + this.dependenciesUpToDate[index] = DependencyState.Ok; if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) - for (const f of cycles.pop()!) - this.dependenciesUpToDate.set(f, true); // update result for files that had a circular dependency on this one - } else { - const parentCircularDep = circularDependenciesQueue[circularDependenciesQueue.length - 1]; - if (parentCircularDep === Number.MAX_SAFE_INTEGER) { - cycles[cycles.length - 1].add(fileName); // parent had no cycle, keep the existing one - } else if (!cycles[cycles.length - 1].has(fileName)) { - const currentCycle = cycles.pop()!; - const previousCycle = cycles[cycles.length - 1]; - previousCycle.add(fileName); - for (const f of currentCycle) - previousCycle.add(f); // merge cycles - } - if (earliestCircularDependency < circularDependenciesQueue[circularDependenciesQueue.length - 1]) - circularDependenciesQueue[circularDependenciesQueue.length - 1] = earliestCircularDependency; + for (const dep of cycles.pop()!) // cycle ends here + this.dependenciesUpToDate[dep] = DependencyState.Ok; // update result for files that had a circular dependency on this one } - if (fileNameQueue.length === 0) + if (parents.length === 0) return true; } } @@ -312,6 +324,7 @@ class ProgramStateImpl implements ProgramState { return { files, lookup, + v: STATE_VERSION, ts: ts.version, global: this.sortByHash(this.resolver.getFilesAffectingGlobalScope()).map(mapToIndex), options: this.optionsHash, @@ -325,40 +338,60 @@ class ProgramStateImpl implements ProgramState { } } -function findCircularDependencyOfCycle( - fileNameQueue: readonly string[], - childCounts: readonly number[], - circularDependencies: readonly number[], - cycle: ReadonlySet, -) { - if (circularDependencies[0] !== Number.MAX_SAFE_INTEGER && cycle.has(fileNameQueue[0])) - return circularDependencies[0]; - for (let i = 0, current = 0; i < childCounts.length; ++i) { - current += childCounts[i]; - const dep = circularDependencies[current]; - if (dep !== Number.MAX_SAFE_INTEGER && cycle.has(fileNameQueue[current])) +function findCircularDependencyOfCycle(parents: readonly number[], circularDependencies: readonly number[], cycle: ReadonlySet) { + for (let i = 0; i < parents.length; ++i) { + const dep = circularDependencies[i]; + if (dep !== Number.MAX_SAFE_INTEGER && cycle.has(parents[i])) return dep; } throw new Error('should never happen'); } -function findParent(stateQueue: readonly number[], childCounts: readonly number[], needle: number): number { - if (stateQueue[0] === needle) - return 0; - for (let i = 0, current = 0; i < childCounts.length; ++i) { - current += childCounts[i]; - if (stateQueue[current] === needle) - return current; +function setCircularDependency(parents: readonly number[], circularDependencies: number[], self: number, cycles: Array>, earliestCircularDependency: number) { + let cyclesToMerge = 0 + for (let i = circularDependencies.length - 1, inCycle = false; i >= earliestCircularDependency; --i) { + const dep = circularDependencies[i]; + if (dep === Number.MAX_SAFE_INTEGER) { + inCycle = false; + } else { + if (!inCycle) { + ++cyclesToMerge; + inCycle = true; + } + if (dep === i) { + inCycle = false; // if cycle ends here, the next parent might start a new one + } else if (dep <= earliestCircularDependency) { + earliestCircularDependency = dep; + break; + } + } + } + let targetCycle; + if (cyclesToMerge === 0) { + targetCycle = new Set(); + cycles.push(targetCycle); + } else { + targetCycle = cycles[cycles.length - cyclesToMerge]; + while (--cyclesToMerge) + for (const d of cycles.pop()!) + targetCycle.add(d); + } + targetCycle.add(self); + for (let i = circularDependencies.length - 1; i >= earliestCircularDependency; --i) { + targetCycle.add(parents[i]); + circularDependencies[i] = earliestCircularDependency; } - return -1; + return earliestCircularDependency; } -function markSelfAndParentsAsOutdated(fileNameQueue: readonly string[], childCounts: readonly number[], results: Map) { - results.set(fileNameQueue[0], false); - for (let i = 0, current = 0; i < childCounts.length; ++i) { - current += childCounts[i]; - results.set(fileNameQueue[current], false); +function markAsOutdated(parents: readonly number[], index: number, cycles: ReadonlyArray>,results: Uint8Array) { + results[index] = DependencyState.Outdated; + for (index of parents) { + results[index] = DependencyState.Outdated; } + for (const cycle of cycles) + for (index of cycle) + results[index] = DependencyState.Outdated; return false; } diff --git a/packages/ymir/src/index.ts b/packages/ymir/src/index.ts index 2a5eef7b3..f0895f77b 100644 --- a/packages/ymir/src/index.ts +++ b/packages/ymir/src/index.ts @@ -523,6 +523,8 @@ export interface StatePersistence { export abstract class StatePersistence {} export interface StaticProgramState { + /** Version of the cache format */ + readonly v: number; /** TypeScript version */ readonly ts: string; /** Hash of compilerOptions */ From 070dc38c25a9cd295a6265ab1075ff0a6e2b2328 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 28 Jan 2021 20:45:43 +0100 Subject: [PATCH 06/37] minor adjustments --- packages/wotan/src/runner.ts | 12 ++++++------ packages/wotan/src/services/program-state.ts | 20 ++++++++++---------- packages/ymir/package.json | 1 - 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index c48fb40ef..709ff98c5 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -153,12 +153,12 @@ export class Runner { } else { summary = { findings: resultFromCache ?? this.linter.getFindings( - sourceFile, - effectiveConfig, - factory, - mapped?.processor, - linterOptions, - ), + sourceFile, + effectiveConfig, + factory, + mapped?.processor, + linterOptions, + ), fixes: 0, content: originalContent, }; diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 293d48f8c..3655a6202 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -124,9 +124,9 @@ class ProgramStateImpl implements ProgramState { if (oldState === undefined) return; const relative = this.getRelativePath(fileName); - if (!(relative in oldState.lookup)) - return; const index = oldState.lookup[relative]; + if (index === undefined) + return; const old = oldState.files[index]; if ( old.result === undefined || @@ -158,10 +158,8 @@ class ProgramStateImpl implements ProgramState { if (oldState === undefined) return false; const relative = this.getRelativePath(fileName); - if (!(relative in oldState.lookup)) - return false; const index = oldState.lookup[relative]; - if (oldState.files[index].hash !== this.getFileHash(fileName)) + if (index === undefined || oldState.files[index].hash !== this.getFileHash(fileName)) return false; switch (this.dependenciesUpToDate[index]) { case DependencyState.Unknown: @@ -304,12 +302,14 @@ class ProgramStateImpl implements ProgramState { for (let i = 0; i < sourceFiles.length; ++i) lookup[this.getRelativePath(sourceFiles[i].fileName)] = i; for (const file of sourceFiles) { - const relativePath = this.relativePathNames.get(file.fileName)!; let results = this.fileResults.get(file.fileName); - if (results === undefined && oldState !== undefined && relativePath in oldState.lookup) { - const old = oldState.files[oldState.lookup[relativePath]]; - if (old.result !== undefined) - results = old; + if (results === undefined && oldState !== undefined) { + const index = oldState.lookup[this.relativePathNames.get(file.fileName)!]; + if (index !== undefined) { + const old = oldState.files[index]; + if (old.result !== undefined) + results = old; + } } if (results !== undefined && !this.isFileUpToDate(file.fileName)) { log('Discarding outdated results for %s', file.fileName); diff --git a/packages/ymir/package.json b/packages/ymir/package.json index 4dfa8660b..0d0b1ce20 100644 --- a/packages/ymir/package.json +++ b/packages/ymir/package.json @@ -27,7 +27,6 @@ "inversify": "^5.0.0", "reflect-metadata": "^0.1.12", "tslib": "^2.0.0", - "tsutils": "^3.20.0" }, "devDependencies": { "tsutils": "^3.5.0" From 318f24c26d1655cdd79ec6d0471f2b88b643bbce Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 30 Jan 2021 14:12:06 +0100 Subject: [PATCH 07/37] make it build again --- baselines/packages/wotan/api/packlist.txt | 9 +++++ .../packages/wotan/api/src/argparse.d.ts | 1 + baselines/packages/wotan/api/src/linter.d.ts | 4 +- baselines/packages/wotan/api/src/runner.d.ts | 4 +- .../services/default/state-persistence.d.ts | 7 ++++ .../api/src/services/dependency-resolver.d.ts | 10 +++++ .../wotan/api/src/services/program-state.d.ts | 23 +++++++++++ baselines/packages/ymir/api/src/index.d.ts | 39 +++++++++++++++++++ packages/mimir/src/rules/no-duplicate-case.ts | 4 +- packages/mithotyn/index.ts | 2 +- packages/valtyr/src/configuration-provider.ts | 2 +- packages/wotan/src/commands/save.ts | 1 + .../src/services/default/state-persistence.ts | 2 +- packages/wotan/src/services/program-state.ts | 21 ++++++---- packages/wotan/test/argparse.spec.ts | 37 ++++++++++++++++++ packages/wotan/test/commands.spec.ts | 12 ++++++ packages/wotan/test/runner.spec.ts | 14 +++++++ packages/ymir/package.json | 2 +- 18 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 baselines/packages/wotan/api/src/services/default/state-persistence.d.ts create mode 100644 baselines/packages/wotan/api/src/services/dependency-resolver.d.ts create mode 100644 baselines/packages/wotan/api/src/services/program-state.d.ts diff --git a/baselines/packages/wotan/api/packlist.txt b/baselines/packages/wotan/api/packlist.txt index 2e8802d11..c8064c366 100644 --- a/baselines/packages/wotan/api/packlist.txt +++ b/baselines/packages/wotan/api/packlist.txt @@ -93,12 +93,21 @@ src/services/default/resolver.js.map src/services/default/rule-loader-host.d.ts src/services/default/rule-loader-host.js src/services/default/rule-loader-host.js.map +src/services/default/state-persistence.d.ts +src/services/default/state-persistence.js +src/services/default/state-persistence.js.map +src/services/dependency-resolver.d.ts +src/services/dependency-resolver.js +src/services/dependency-resolver.js.map src/services/formatter-loader.d.ts src/services/formatter-loader.js src/services/formatter-loader.js.map src/services/processor-loader.d.ts src/services/processor-loader.js src/services/processor-loader.js.map +src/services/program-state.d.ts +src/services/program-state.js +src/services/program-state.js.map src/services/rule-loader.d.ts src/services/rule-loader.js src/services/rule-loader.js.map diff --git a/baselines/packages/wotan/api/src/argparse.d.ts b/baselines/packages/wotan/api/src/argparse.d.ts index 0f5494a7a..c9480f2ca 100644 --- a/baselines/packages/wotan/api/src/argparse.d.ts +++ b/baselines/packages/wotan/api/src/argparse.d.ts @@ -12,6 +12,7 @@ export declare const GLOBAL_OPTIONS_SPEC: { exclude: OptionParser.ParseFunction; project: OptionParser.ParseFunction; references: OptionParser.ParseFunction; + cache: OptionParser.ParseFunction; formatter: OptionParser.ParseFunction; fix: OptionParser.ParseFunction; extensions: OptionParser.ParseFunction; diff --git a/baselines/packages/wotan/api/src/linter.d.ts b/baselines/packages/wotan/api/src/linter.d.ts index 0cd38a475..d4034e5ed 100644 --- a/baselines/packages/wotan/api/src/linter.d.ts +++ b/baselines/packages/wotan/api/src/linter.d.ts @@ -20,5 +20,7 @@ export declare type UpdateFileCallback = (content: string, range: ts.TextChangeR export declare class Linter { constructor(ruleLoader: RuleLoader, logger: MessageHandler, deprecationHandler: DeprecationHandler, filterFactory: FindingFilterFactory); lintFile(file: ts.SourceFile, config: EffectiveConfiguration, programOrFactory?: ProgramFactory | ts.Program, options?: LinterOptions): ReadonlyArray; - lintAndFix(file: ts.SourceFile, content: string, config: EffectiveConfiguration, updateFile: UpdateFileCallback, iterations?: number, programFactory?: ProgramFactory, processor?: AbstractProcessor, options?: LinterOptions): LintAndFixFileResult; + lintAndFix(file: ts.SourceFile, content: string, config: EffectiveConfiguration, updateFile: UpdateFileCallback, iterations?: number, programFactory?: ProgramFactory, processor?: AbstractProcessor, options?: LinterOptions, + /** Initial set of findings from a cache. If provided, the initial linting is skipped and these findings are used for fixing. */ + findings?: readonly Finding[]): LintAndFixFileResult; } diff --git a/baselines/packages/wotan/api/src/runner.d.ts b/baselines/packages/wotan/api/src/runner.d.ts index 019fa5ee5..ac0b6fc1d 100644 --- a/baselines/packages/wotan/api/src/runner.d.ts +++ b/baselines/packages/wotan/api/src/runner.d.ts @@ -4,6 +4,7 @@ import * as ts from 'typescript'; import { ProcessorLoader } from './services/processor-loader'; import { CachedFileSystem } from './services/cached-file-system'; import { ConfigurationManager } from './services/configuration-manager'; +import { ProgramStateFactory } from './services/program-state'; export interface LintOptions { config: string | undefined; files: ReadonlyArray; @@ -13,9 +14,10 @@ export interface LintOptions { fix: boolean | number; extensions: ReadonlyArray | undefined; reportUselessDirectives: Severity | boolean | undefined; + cache: boolean; } export declare class Runner { - constructor(fs: CachedFileSystem, configManager: ConfigurationManager, linter: Linter, processorLoader: ProcessorLoader, directories: DirectoryService, logger: MessageHandler, filterFactory: FileFilterFactory); + constructor(fs: CachedFileSystem, configManager: ConfigurationManager, linter: Linter, processorLoader: ProcessorLoader, directories: DirectoryService, logger: MessageHandler, filterFactory: FileFilterFactory, programStateFactory: ProgramStateFactory); lintCollection(options: LintOptions): LintResult; } declare module 'typescript' { diff --git a/baselines/packages/wotan/api/src/services/default/state-persistence.d.ts b/baselines/packages/wotan/api/src/services/default/state-persistence.d.ts new file mode 100644 index 000000000..cd6884db6 --- /dev/null +++ b/baselines/packages/wotan/api/src/services/default/state-persistence.d.ts @@ -0,0 +1,7 @@ +import { CachedFileSystem } from '../cached-file-system'; +import { StatePersistence, StaticProgramState } from '@fimbul/ymir'; +export declare class DefaultStatePersistence implements StatePersistence { + constructor(fs: CachedFileSystem); + loadState(project: string): StaticProgramState | undefined; + saveState(project: string, state: StaticProgramState): void; +} diff --git a/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts b/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts new file mode 100644 index 000000000..95b69be7f --- /dev/null +++ b/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts @@ -0,0 +1,10 @@ +import * as ts from 'typescript'; +import { ProjectHost } from '../project-host'; +export interface DependencyResolver { + update(program: ts.Program, updatedFile: string): void; + getDependencies(fileName: string): ReadonlyMap; + getFilesAffectingGlobalScope(): readonly string[]; +} +export declare class DependencyResolverFactory { + create(host: ProjectHost, program: ts.Program): DependencyResolver; +} diff --git a/baselines/packages/wotan/api/src/services/program-state.d.ts b/baselines/packages/wotan/api/src/services/program-state.d.ts new file mode 100644 index 000000000..a54caf212 --- /dev/null +++ b/baselines/packages/wotan/api/src/services/program-state.d.ts @@ -0,0 +1,23 @@ +import * as ts from 'typescript'; +import { DependencyResolver, DependencyResolverFactory } from './dependency-resolver'; +import { EffectiveConfiguration, Finding, ReducedConfiguration, StatePersistence } from '@fimbul/ymir'; +import { ProjectHost } from '../project-host'; +export interface ProgramState { + update(program: ts.Program, updatedFile: string): void; + getUpToDateResult(fileName: string, config: EffectiveConfiguration): readonly Finding[] | undefined; + setFileResult(fileName: string, config: EffectiveConfiguration, result: readonly Finding[]): void; + save(): void; +} +export declare class ProgramStateFactory { + constructor(resolverFactory: DependencyResolverFactory, statePersistence: StatePersistence); + create(program: ts.Program, host: ProjectHost, tsconfigPath: string): ProgramStateImpl; +} +declare const oldStateSymbol: unique symbol; +declare class ProgramStateImpl implements ProgramState { + constructor(host: ts.CompilerHost, program: ts.Program, resolver: DependencyResolver, statePersistence: StatePersistence, project: string); + update(program: ts.Program, updatedFile: string): void; + getUpToDateResult(fileName: string, config: ReducedConfiguration): readonly Finding[] | undefined; + setFileResult(fileName: string, config: ReducedConfiguration, result: ReadonlyArray): void; + save(): void; +} +export {}; diff --git a/baselines/packages/ymir/api/src/index.d.ts b/baselines/packages/ymir/api/src/index.d.ts index fef9657e3..96f58b52a 100644 --- a/baselines/packages/ymir/api/src/index.d.ts +++ b/baselines/packages/ymir/api/src/index.d.ts @@ -363,3 +363,42 @@ export interface FileFilter { /** @returns `true` if the file should be linted, false if it should be filtered out. Intended for use in `Array.prototype.filter`. */ filter(file: ts.SourceFile): boolean; } +export interface StatePersistence { + loadState(project: string): StaticProgramState | undefined; + saveState(project: string, state: StaticProgramState): void; +} +export declare abstract class StatePersistence { +} +export interface StaticProgramState { + /** Version of the cache format */ + readonly v: number; + /** TypeScript version */ + readonly ts: string; + /** Hash of compilerOptions */ + readonly options: string; + /** Maps filename to index in 'files' array */ + readonly lookup: Readonly>; + /** Index of files that affect global scope */ + readonly global: readonly number[]; + /** Information about all files in the program */ + readonly files: readonly StaticProgramState.FileState[]; +} +export declare namespace StaticProgramState { + interface FileState { + /** Hash of file contents */ + readonly hash: string; + /** + * Key: module specifier as referenced in the file, order may be random + * Value: - `null` if dependency could not be resolved + * - List of files (or rather their index) that the module specifier resolves to. + * That is the actual file at that path and/or files containing `declare module "..."` for that module specifier. + * May contain the current file. + * This list is ordered by the hash of the files ascending, + */ + readonly dependencies: Readonly>; + /** The list of findings if this file has up-to-date results */ + readonly result?: readonly Finding[]; + /** Hash of the configuration used to produce `result` for this file */ + readonly config?: string; + } +} diff --git a/packages/mimir/src/rules/no-duplicate-case.ts b/packages/mimir/src/rules/no-duplicate-case.ts index 21407eb95..5227f50be 100644 --- a/packages/mimir/src/rules/no-duplicate-case.ts +++ b/packages/mimir/src/rules/no-duplicate-case.ts @@ -68,7 +68,7 @@ export class Rule extends AbstractRule { if (isBigIntLiteral(node)) return [formatPrimitive(prefixFn({base10Value: node.text.slice(0, -1), negative: false}))]; if (node.kind === ts.SyntaxKind.NullKeyword) - return [formatPrimitive(prefixFn(null))]; // tslint:disable-line:no-null-keyword + return [formatPrimitive(prefixFn(null))]; if (isIdentifier(node) && node.originalKeywordKind === ts.SyntaxKind.UndefinedKeyword) return [formatPrimitive(prefixFn(undefined))]; if (node.kind === ts.SyntaxKind.TrueKeyword) @@ -89,7 +89,7 @@ export class Rule extends AbstractRule { } else if (t.flags & ts.TypeFlags.Undefined) { result.add(formatPrimitive(prefixFn(undefined))); } else if (t.flags & ts.TypeFlags.Null) { - result.add(formatPrimitive(prefixFn(null))); // tslint:disable-line:no-null-keyword + result.add(formatPrimitive(prefixFn(null))); } else { return []; } diff --git a/packages/mithotyn/index.ts b/packages/mithotyn/index.ts index 196f7cf85..91ae78096 100644 --- a/packages/mithotyn/index.ts +++ b/packages/mithotyn/index.ts @@ -58,7 +58,7 @@ function createProxy( interceptor: Partial, log: (m: string) => void, ) { - const proxy = Object.create(null); // tslint:disable-line:no-null-keyword + const proxy = Object.create(null); for (const method of Object.keys(ls)) { if (typeof (interceptor)[method] === 'function') { proxy[method] = (...args: any[]) => { diff --git a/packages/valtyr/src/configuration-provider.ts b/packages/valtyr/src/configuration-provider.ts index 9ddc6dde1..44f64d231 100644 --- a/packages/valtyr/src/configuration-provider.ts +++ b/packages/valtyr/src/configuration-provider.ts @@ -44,7 +44,7 @@ export class TslintConfigurationProvider implements ConfigurationProvider { fileName = path.dirname(fileName); let result = this.cache.get(fileName); if (result === undefined && !this.cache.has(fileName)) { - result = TSLint.Configuration.findConfigurationPath(null, fileName); // tslint:disable-line:no-null-keyword + result = TSLint.Configuration.findConfigurationPath(null, fileName); const {root} = path.parse(fileName); // prevent infinite loop when result is on different drive const configDirname = result === undefined || root !== path.parse(result).root ? root : path.dirname(result); diff --git a/packages/wotan/src/commands/save.ts b/packages/wotan/src/commands/save.ts index 74943504b..a0a46882d 100644 --- a/packages/wotan/src/commands/save.ts +++ b/packages/wotan/src/commands/save.ts @@ -21,6 +21,7 @@ class SaveCommandRunner extends AbstractCommandRunner { { ...this.options, ...config, + cache: config.cache || undefined, fix: config.fix || undefined, reportUselessDirectives: config.reportUselessDirectives || undefined, references: config.references || undefined, diff --git a/packages/wotan/src/services/default/state-persistence.ts b/packages/wotan/src/services/default/state-persistence.ts index b41f5ef9f..2c3608c41 100644 --- a/packages/wotan/src/services/default/state-persistence.ts +++ b/packages/wotan/src/services/default/state-persistence.ts @@ -39,7 +39,7 @@ export class DefaultStatePersistence implements StatePersistence { const fileName = buildFilename(project); log("Writing cache '%s'", fileName); try { - const content: CacheFileContent = {v: CACHE_VERSION, state}; + const content: CacheFileContent = {state, v: CACHE_VERSION}; this.fs.writeFile(fileName, yaml.dump(content, {indent: 2, sortKeys: true})); } catch { log("Error writing cache '%s'", fileName); diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 3655a6202..a016c469b 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -22,6 +22,7 @@ export interface ProgramState { export class ProgramStateFactory { constructor(private resolverFactory: DependencyResolverFactory, private statePersistence: StatePersistence) {} + // TODO don't depend on ProjectHost public create(program: ts.Program, host: ProjectHost, tsconfigPath: string) { return new ProgramStateImpl(host, program, this.resolverFactory.create(host, program), this.statePersistence, tsconfigPath); } @@ -232,7 +233,7 @@ class ProgramStateImpl implements ProgramState { if (newDepsWithHash[i].hash !== oldDepState.hash) return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); if (!this.assumeChangesOnlyAffectDirectDependencies && fileName !== newDepsWithHash[i].fileName) { - const indexInQueue = parents.indexOf(oldDeps[i]) + const indexInQueue = parents.indexOf(oldDeps[i]); if (indexInQueue === -1) { // no circular dependency fileNameQueue.push(newDepsWithHash[i].fileName); @@ -273,7 +274,8 @@ class ProgramStateImpl implements ProgramState { this.dependenciesUpToDate[index] = DependencyState.Ok; if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) for (const dep of cycles.pop()!) // cycle ends here - this.dependenciesUpToDate[dep] = DependencyState.Ok; // update result for files that had a circular dependency on this one + // update result for files that had a circular dependency on this one + this.dependenciesUpToDate[dep] = DependencyState.Ok; } if (parents.length === 0) return true; @@ -347,8 +349,14 @@ function findCircularDependencyOfCycle(parents: readonly number[], circularDepen throw new Error('should never happen'); } -function setCircularDependency(parents: readonly number[], circularDependencies: number[], self: number, cycles: Array>, earliestCircularDependency: number) { - let cyclesToMerge = 0 +function setCircularDependency( + parents: readonly number[], + circularDependencies: number[], + self: number, + cycles: Array>, + earliestCircularDependency: number, +) { + let cyclesToMerge = 0; for (let i = circularDependencies.length - 1, inCycle = false; i >= earliestCircularDependency; --i) { const dep = circularDependencies[i]; if (dep === Number.MAX_SAFE_INTEGER) { @@ -384,11 +392,10 @@ function setCircularDependency(parents: readonly number[], circularDependencies: return earliestCircularDependency; } -function markAsOutdated(parents: readonly number[], index: number, cycles: ReadonlyArray>,results: Uint8Array) { +function markAsOutdated(parents: readonly number[], index: number, cycles: ReadonlyArray>, results: Uint8Array) { results[index] = DependencyState.Outdated; - for (index of parents) { + for (index of parents) results[index] = DependencyState.Outdated; - } for (const cycle of cycles) for (index of cycle) results[index] = DependencyState.Outdated; diff --git a/packages/wotan/test/argparse.spec.ts b/packages/wotan/test/argparse.spec.ts index 6075cd91c..d96e0ebba 100644 --- a/packages/wotan/test/argparse.spec.ts +++ b/packages/wotan/test/argparse.spec.ts @@ -17,6 +17,7 @@ test('parseGlobalOptions', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, ); @@ -33,6 +34,7 @@ test('parseGlobalOptions', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'ignores excess options', ); @@ -50,6 +52,7 @@ test('parseGlobalOptions', (t) => { fix: 10, extensions: ['.mjs'], reportUselessDirectives: false, + cache: false, }, ); @@ -66,6 +69,7 @@ test('parseGlobalOptions', (t) => { fix: true, extensions: undefined, reportUselessDirectives: false, + cache: false, }, ); @@ -82,6 +86,7 @@ test('parseGlobalOptions', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'invalid values are ignored', ); @@ -112,6 +117,7 @@ test('defaults to lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, ); t.deepEqual( @@ -128,6 +134,7 @@ test('defaults to lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, ); }); @@ -147,6 +154,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'parses modules', ); @@ -165,6 +173,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'treats all arguments after -- as files', ); @@ -183,6 +192,7 @@ test('parses lint command', (t) => { fix: true, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'trims single quotes', ); @@ -201,6 +211,7 @@ test('parses lint command', (t) => { fix: true, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '--fix argument is optional', ); @@ -219,6 +230,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '--fix can be set to false', ); @@ -237,6 +249,7 @@ test('parses lint command', (t) => { fix: true, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '--fix can be set to true', ); @@ -255,6 +268,7 @@ test('parses lint command', (t) => { fix: 10, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '--fix can be set to any number', ); @@ -273,6 +287,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '--project is accumulated', ); @@ -291,6 +306,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '--exclude is accumulated', ); @@ -309,6 +325,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'files can be interspersed, specifying an option multiple times overrides its value', ); @@ -327,6 +344,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '-c specifies config', ); @@ -345,6 +363,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '--config specifies config', ); @@ -363,6 +382,7 @@ test('parses lint command', (t) => { fix: false, extensions: ['.mjs', '.es6', '.esm'], reportUselessDirectives: false, + cache: false, }, '--ext can be comma separated, values are sanitized', ); @@ -381,6 +401,7 @@ test('parses lint command', (t) => { fix: false, extensions: ['.mjs', '.es6'], reportUselessDirectives: false, + cache: false, }, '--ext can occur multiple times', ); @@ -399,6 +420,7 @@ test('parses lint command', (t) => { fix: false, extensions: ['.esm', '.mjs', '.es6'], reportUselessDirectives: false, + cache: false, }, '--ext merges arrays', ); @@ -417,6 +439,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '-r switches project references', ); @@ -435,6 +458,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, '--references switches project references', ); @@ -466,6 +490,7 @@ test('parses lint command', (t) => { fix: true, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'overrides defaults', ); @@ -497,6 +522,7 @@ test('parses lint command', (t) => { fix: 10, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'uses defaults where not overridden', ); @@ -515,6 +541,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: true, + cache: false, }, 'value for --report-useless-directives is optional, default is true', ); @@ -533,6 +560,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: true, + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -551,6 +579,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -569,6 +598,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: true, + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -587,6 +617,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: 'error', + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -605,6 +636,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: 'warning', + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -623,6 +655,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: 'warning', + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -641,6 +674,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: 'suggestion', + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -659,6 +693,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: 'suggestion', + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -677,6 +712,7 @@ test('parses lint command', (t) => { fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, 'only parses severity or boolean as value for --report-useless-directives', ); @@ -721,6 +757,7 @@ test('parses save command', (t) => { fix: 10, extensions: undefined, reportUselessDirectives: false, + cache: false, }, ); }); diff --git a/packages/wotan/test/commands.spec.ts b/packages/wotan/test/commands.spec.ts index ab8b6f859..55745f4b4 100644 --- a/packages/wotan/test/commands.spec.ts +++ b/packages/wotan/test/commands.spec.ts @@ -170,6 +170,7 @@ test('SaveCommand', async (t) => { formatter: undefined, reportUselessDirectives: false, modules: [], + cache: false, }), { content: false, @@ -191,6 +192,7 @@ test('SaveCommand', async (t) => { formatter: undefined, reportUselessDirectives: false, modules: [], + cache: false, }, { project: 'foo', @@ -217,6 +219,7 @@ test('SaveCommand', async (t) => { formatter: undefined, reportUselessDirectives: true, modules: [], + cache: false, }, { project: 'foo.json', @@ -247,6 +250,7 @@ test('SaveCommand', async (t) => { formatter: undefined, reportUselessDirectives: false, modules: [], + cache: false, }, { other: 'foo', @@ -350,6 +354,7 @@ test('LintCommand', async (t) => { fix: true, extensions: undefined, reportUselessDirectives: false, + cache: false, }, container, ), @@ -372,6 +377,7 @@ test('LintCommand', async (t) => { fix: false, extensions: undefined, reportUselessDirectives: true, + cache: false, }, container, ), @@ -401,6 +407,7 @@ ERROR 2:1 useless-line-switch Disable switch has no effect. All specified rule fix: false, extensions: undefined, reportUselessDirectives: true, + cache: false, }, container, ), @@ -430,6 +437,7 @@ ERROR 2:1 useless-line-switch Disable switch has no effect. All specified rule fix: false, extensions: undefined, reportUselessDirectives: 'warning', + cache: false, }, container, ), @@ -459,6 +467,7 @@ WARNING 2:1 useless-line-switch Disable switch has no effect. All specified ru fix: true, extensions: undefined, reportUselessDirectives: true, + cache: false, }, container, ), @@ -485,6 +494,7 @@ WARNING 2:1 useless-line-switch Disable switch has no effect. All specified ru fix: true, extensions: undefined, reportUselessDirectives: false, + cache: false, }, container, ), @@ -517,6 +527,7 @@ ERROR 2:8 no-unused-expression This expression is unused. Did you mean to assi fix: true, extensions: undefined, reportUselessDirectives: false, + cache: false, }, container, ), @@ -543,6 +554,7 @@ ERROR 2:8 no-unused-expression This expression is unused. Did you mean to assi fix: false, extensions: undefined, reportUselessDirectives: false, + cache: false, }, container, ), diff --git a/packages/wotan/test/runner.spec.ts b/packages/wotan/test/runner.spec.ts index b6e726ed6..b06eec437 100644 --- a/packages/wotan/test/runner.spec.ts +++ b/packages/wotan/test/runner.spec.ts @@ -19,6 +19,7 @@ test('throws error on non-existing file', (t) => { const runner = container.get(Runner); t.throws( () => Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [ 'test/fixtures/invalid.js', // exists @@ -44,6 +45,7 @@ test('throws error on file not included in project', (t) => { const runner = container.get(Runner); t.throws( () => Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [ 'non-existent.js', // does not exist, but is excluded @@ -69,6 +71,7 @@ test('handles absolute paths with file system specific path separator', (t) => { container.load(createCoreModule({}), createDefaultModule()); const runner = container.get(Runner); const result = Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [ path.resolve('packages/wotan/test/project/setup/test.ts'), @@ -105,6 +108,7 @@ test('throws if no tsconfig.json can be found', (t) => { const {root} = path.parse(process.cwd()); t.throws( () => Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], @@ -120,6 +124,7 @@ test('throws if no tsconfig.json can be found', (t) => { const dir = path.join(__dirname, 'non-existent'); t.throws( () => Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], @@ -134,6 +139,7 @@ test('throws if no tsconfig.json can be found', (t) => { t.throws( () => Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], @@ -192,6 +198,7 @@ test('reports warnings while parsing tsconfig.json', (t) => { const runner = container.get(Runner); Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], @@ -205,6 +212,7 @@ test('reports warnings while parsing tsconfig.json', (t) => { warning = ''; Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], @@ -218,6 +226,7 @@ test('reports warnings while parsing tsconfig.json', (t) => { warning = ''; Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], @@ -233,6 +242,7 @@ test('reports warnings while parsing tsconfig.json', (t) => { warning = ''; Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], @@ -344,6 +354,7 @@ test.skip('excludes symlinked typeRoots', (t) => { container.load(createCoreModule({}), createDefaultModule()); const runner = container.get(Runner); const result = Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], @@ -371,6 +382,7 @@ test('works with absolute and relative paths', (t) => { function testRunner(project: boolean) { const result = Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [ unixifyPath(path.resolve('packages/wotan/test/fixtures/paths/a.ts')), @@ -408,6 +420,7 @@ test('normalizes globs', (t) => { function testRunner(project: boolean) { const result = Array.from(runner.lintCollection({ + cache: false, config: undefined, files: [ '../paths/a.ts', @@ -435,6 +448,7 @@ test('supports linting multiple (overlapping) projects in one run', (t) => { const result = Array.from( runner.lintCollection({ + cache: false, config: undefined, files: [], exclude: [], diff --git a/packages/ymir/package.json b/packages/ymir/package.json index 0d0b1ce20..edeeece59 100644 --- a/packages/ymir/package.json +++ b/packages/ymir/package.json @@ -26,7 +26,7 @@ "dependencies": { "inversify": "^5.0.0", "reflect-metadata": "^0.1.12", - "tslib": "^2.0.0", + "tslib": "^2.0.0" }, "devDependencies": { "tsutils": "^3.5.0" From 1b73a0a3a1b2644b7a3b257f36284c194dad7b83 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 30 Jan 2021 18:48:48 +0100 Subject: [PATCH 08/37] remove dependency on ProjectHost --- .../api/src/services/dependency-resolver.d.ts | 4 +- .../wotan/api/src/services/program-state.d.ts | 8 ++-- baselines/packages/ymir/api/src/index.d.ts | 2 + .../wotan/src/services/dependency-resolver.ts | 18 ++++----- packages/wotan/src/services/program-state.ts | 37 +++++++++++++------ packages/ymir/src/index.ts | 2 + 6 files changed, 44 insertions(+), 27 deletions(-) diff --git a/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts b/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts index 95b69be7f..917bb6f4f 100644 --- a/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts +++ b/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts @@ -1,10 +1,10 @@ import * as ts from 'typescript'; -import { ProjectHost } from '../project-host'; export interface DependencyResolver { update(program: ts.Program, updatedFile: string): void; getDependencies(fileName: string): ReadonlyMap; getFilesAffectingGlobalScope(): readonly string[]; } +export declare type DependencyResolverHost = Required>; export declare class DependencyResolverFactory { - create(host: ProjectHost, program: ts.Program): DependencyResolver; + create(host: DependencyResolverHost, program: ts.Program): DependencyResolver; } diff --git a/baselines/packages/wotan/api/src/services/program-state.d.ts b/baselines/packages/wotan/api/src/services/program-state.d.ts index a54caf212..9fc97aab6 100644 --- a/baselines/packages/wotan/api/src/services/program-state.d.ts +++ b/baselines/packages/wotan/api/src/services/program-state.d.ts @@ -1,7 +1,6 @@ import * as ts from 'typescript'; -import { DependencyResolver, DependencyResolverFactory } from './dependency-resolver'; +import { DependencyResolver, DependencyResolverFactory, DependencyResolverHost } from './dependency-resolver'; import { EffectiveConfiguration, Finding, ReducedConfiguration, StatePersistence } from '@fimbul/ymir'; -import { ProjectHost } from '../project-host'; export interface ProgramState { update(program: ts.Program, updatedFile: string): void; getUpToDateResult(fileName: string, config: EffectiveConfiguration): readonly Finding[] | undefined; @@ -10,11 +9,12 @@ export interface ProgramState { } export declare class ProgramStateFactory { constructor(resolverFactory: DependencyResolverFactory, statePersistence: StatePersistence); - create(program: ts.Program, host: ProjectHost, tsconfigPath: string): ProgramStateImpl; + create(program: ts.Program, host: ProgramStateHost & DependencyResolverHost, tsconfigPath: string): ProgramStateImpl; } +export declare type ProgramStateHost = Pick; declare const oldStateSymbol: unique symbol; declare class ProgramStateImpl implements ProgramState { - constructor(host: ts.CompilerHost, program: ts.Program, resolver: DependencyResolver, statePersistence: StatePersistence, project: string); + constructor(host: ProgramStateHost, program: ts.Program, resolver: DependencyResolver, statePersistence: StatePersistence, project: string); update(program: ts.Program, updatedFile: string): void; getUpToDateResult(fileName: string, config: ReducedConfiguration): readonly Finding[] | undefined; setFileResult(fileName: string, config: ReducedConfiguration, result: ReadonlyArray): void; diff --git a/baselines/packages/ymir/api/src/index.d.ts b/baselines/packages/ymir/api/src/index.d.ts index 96f58b52a..d300f7fbb 100644 --- a/baselines/packages/ymir/api/src/index.d.ts +++ b/baselines/packages/ymir/api/src/index.d.ts @@ -374,6 +374,8 @@ export interface StaticProgramState { readonly v: number; /** TypeScript version */ readonly ts: string; + /** Whether the state was created using case-sensitive file names */ + readonly cs: boolean; /** Hash of compilerOptions */ readonly options: string; /** Maps filename to index in 'files' array */ diff --git a/packages/wotan/src/services/dependency-resolver.ts b/packages/wotan/src/services/dependency-resolver.ts index 57cc16de0..7dab392b2 100644 --- a/packages/wotan/src/services/dependency-resolver.ts +++ b/packages/wotan/src/services/dependency-resolver.ts @@ -1,9 +1,8 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; -import { isModuleDeclaration, isNamespaceExportDeclaration, findImports, ImportKind, isCompilerOptionEnabled } from 'tsutils'; +import { isModuleDeclaration, isNamespaceExportDeclaration, findImports, ImportKind } from 'tsutils'; import { resolveCachedResult, getOutputFileNamesOfProjectReference, iterateProjectReferences } from '../utils'; import bind from 'bind-decorator'; -import { ProjectHost } from '../project-host'; export interface DependencyResolver { update(program: ts.Program, updatedFile: string): void; @@ -11,9 +10,11 @@ export interface DependencyResolver { getFilesAffectingGlobalScope(): readonly string[]; } +export type DependencyResolverHost = Required>; + @injectable() export class DependencyResolverFactory { - public create(host: ProjectHost, program: ts.Program): DependencyResolver { + public create(host: DependencyResolverHost, program: ts.Program): DependencyResolver { return new DependencyResolverImpl(host, program); } } @@ -33,7 +34,7 @@ class DependencyResolverImpl implements DependencyResolver { private state: DependencyResolverState | undefined = undefined; - constructor(private host: ProjectHost, private program: ts.Program) {} + constructor(private host: DependencyResolverHost, private program: ts.Program) {} public update(program: ts.Program, updatedFile: string) { this.state = undefined; @@ -138,7 +139,7 @@ class DependencyResolverImpl implements DependencyResolver { @bind private collectMetaDataForFile(fileName: string) { - return collectFileMetadata(this.program.getSourceFile(fileName)!, this.compilerOptions); + return collectFileMetadata(this.program.getSourceFile(fileName)!); } @bind @@ -156,7 +157,7 @@ class DependencyResolverImpl implements DependencyResolver { return result; this.fileToProjectReference ??= createProjectReferenceMap(this.program.getResolvedProjectReferences()); const arr = Array.from(references); - const resolved = this.host.resolveModuleNames(arr, fileName, undefined, this.fileToProjectReference.get(fileName)); + const resolved = this.host.resolveModuleNames(arr, fileName, undefined, this.fileToProjectReference.get(fileName), this.compilerOptions); for (let i = 0; i < resolved.length; ++i) result.set(arr[i], resolved[i]?.resolvedFileName ?? null); return result; @@ -206,7 +207,7 @@ interface MetaData { isExternalModule: boolean; } -function collectFileMetadata(sourceFile: ts.SourceFile, compilerOptions: ts.CompilerOptions): MetaData { +function collectFileMetadata(sourceFile: ts.SourceFile): MetaData { let affectsGlobalScope: boolean | undefined; const ambientModules = new Set(); const isExternalModule = ts.isExternalModule(sourceFile); @@ -216,8 +217,7 @@ function collectFileMetadata(sourceFile: ts.SourceFile, compilerOptions: ts.Comp } else if (isModuleDeclaration(statement) && statement.name.kind === ts.SyntaxKind.StringLiteral) { ambientModules.add(statement.name.text); } else if (isNamespaceExportDeclaration(statement)) { - if (isCompilerOptionEnabled(compilerOptions, 'allowUmdGlobalAccess')) - affectsGlobalScope = true; + affectsGlobalScope = true; } else if (affectsGlobalScope === undefined) { // files that only consist of ambient modules do not affect global scope affectsGlobalScope = !isExternalModule; } diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index a016c469b..2bbf61915 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -1,11 +1,10 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; -import { DependencyResolver, DependencyResolverFactory } from './dependency-resolver'; +import { DependencyResolver, DependencyResolverFactory, DependencyResolverHost } from './dependency-resolver'; import { resolveCachedResult, djb2 } from '../utils'; import bind from 'bind-decorator'; import { EffectiveConfiguration, Finding, ReducedConfiguration, StatePersistence, StaticProgramState } from '@fimbul/ymir'; import debug = require('debug'); -import { ProjectHost } from '../project-host'; import { isCompilerOptionEnabled } from 'tsutils'; import * as path from 'path'; @@ -22,12 +21,13 @@ export interface ProgramState { export class ProgramStateFactory { constructor(private resolverFactory: DependencyResolverFactory, private statePersistence: StatePersistence) {} - // TODO don't depend on ProjectHost - public create(program: ts.Program, host: ProjectHost, tsconfigPath: string) { + public create(program: ts.Program, host: ProgramStateHost & DependencyResolverHost, tsconfigPath: string) { return new ProgramStateImpl(host, program, this.resolverFactory.create(host, program), this.statePersistence, tsconfigPath); } } +export type ProgramStateHost = Pick; + interface FileResults { readonly config: string; readonly result: ReadonlyArray; @@ -56,7 +56,7 @@ class ProgramStateImpl implements ProgramState { private dependenciesUpToDate: Uint8Array; constructor( - private host: ts.CompilerHost, + private host: ProgramStateHost, private program: ts.Program, private resolver: DependencyResolver, private statePersistence: StatePersistence, @@ -67,7 +67,7 @@ class ProgramStateImpl implements ProgramState { this[oldStateSymbol] = undefined; this.dependenciesUpToDate = new Uint8Array(0); } else { - this[oldStateSymbol] = oldState; + this[oldStateSymbol] = this.remapFileNames(oldState); this.dependenciesUpToDate = new Uint8Array(oldState.files.length); } } @@ -124,8 +124,7 @@ class ProgramStateImpl implements ProgramState { const oldState = this.tryReuseOldState(); if (oldState === undefined) return; - const relative = this.getRelativePath(fileName); - const index = oldState.lookup[relative]; + const index = this.lookupFileIndex(fileName, oldState); if (index === undefined) return; const old = oldState.files[index]; @@ -145,7 +144,7 @@ class ProgramStateImpl implements ProgramState { log('File %s is outdated, merging current state into old state', fileName); // we need to create a state where the file is up-to-date // so we replace the old state with the current state - // this includes all results from old state that were still up-to-date and all file results if they were still valid + // this includes all results from old state that are still up-to-date and all file results if they are still valid const newState = this[oldStateSymbol] = this.aggregate(); this.recheckOldState = false; this.fileResults = new Map(); @@ -158,8 +157,7 @@ class ProgramStateImpl implements ProgramState { const oldState = this.tryReuseOldState(); if (oldState === undefined) return false; - const relative = this.getRelativePath(fileName); - const index = oldState.lookup[relative]; + const index = this.lookupFileIndex(fileName, oldState); if (index === undefined || oldState.files[index].hash !== this.getFileHash(fileName)) return false; switch (this.dependenciesUpToDate[index]) { @@ -306,7 +304,7 @@ class ProgramStateImpl implements ProgramState { for (const file of sourceFiles) { let results = this.fileResults.get(file.fileName); if (results === undefined && oldState !== undefined) { - const index = oldState.lookup[this.relativePathNames.get(file.fileName)!]; + const index = this.lookupFileIndex(file.fileName, oldState); if (index !== undefined) { const old = oldState.files[index]; if (old.result !== undefined) @@ -328,6 +326,7 @@ class ProgramStateImpl implements ProgramState { lookup, v: STATE_VERSION, ts: ts.version, + cs: this.host.useCaseSensitiveFileNames(), global: this.sortByHash(this.resolver.getFilesAffectingGlobalScope()).map(mapToIndex), options: this.optionsHash, }; @@ -338,6 +337,20 @@ class ProgramStateImpl implements ProgramState { .map((f) => ({fileName: f, hash: this.getFileHash(f)})) .sort(compareHashKey); } + + private lookupFileIndex(fileName: string, oldState: StaticProgramState): number | undefined { + return oldState.lookup[this.host.getCanonicalFileName(this.getRelativePath(fileName))]; + } + + private remapFileNames(oldState: StaticProgramState): StaticProgramState { + // only need to remap if oldState is case sensitive and current host is case insensitive + if (!oldState.cs || this.host.useCaseSensitiveFileNames()) + return oldState; + const lookup: Record = {}; + for (const [key, value] of Object.entries(oldState.lookup)) + lookup[this.host.getCanonicalFileName(key)] = value; + return {...oldState, lookup}; + } } function findCircularDependencyOfCycle(parents: readonly number[], circularDependencies: readonly number[], cycle: ReadonlySet) { diff --git a/packages/ymir/src/index.ts b/packages/ymir/src/index.ts index f0895f77b..2229ada76 100644 --- a/packages/ymir/src/index.ts +++ b/packages/ymir/src/index.ts @@ -527,6 +527,8 @@ export interface StaticProgramState { readonly v: number; /** TypeScript version */ readonly ts: string; + /** Whether the state was created using case-sensitive file names */ + readonly cs: boolean; /** Hash of compilerOptions */ readonly options: string; /** Maps filename to index in 'files' array */ From ef476f993beadf985f3ce5fed24e9b8a8a818bb9 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 30 Jan 2021 21:50:43 +0100 Subject: [PATCH 09/37] handle useSourceOfProjectReferenceRedirect --- .../api/src/services/dependency-resolver.d.ts | 4 +- baselines/packages/ymir/api/src/index.d.ts | 4 +- packages/wotan/src/project-host.ts | 10 +++- .../wotan/src/services/default/file-filter.ts | 11 ++++ .../wotan/src/services/dependency-resolver.ts | 54 +++++++++++++++---- packages/wotan/src/services/program-state.ts | 4 +- packages/ymir/src/index.ts | 4 +- 7 files changed, 75 insertions(+), 16 deletions(-) diff --git a/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts b/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts index 917bb6f4f..257df3670 100644 --- a/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts +++ b/baselines/packages/wotan/api/src/services/dependency-resolver.d.ts @@ -4,7 +4,9 @@ export interface DependencyResolver { getDependencies(fileName: string): ReadonlyMap; getFilesAffectingGlobalScope(): readonly string[]; } -export declare type DependencyResolverHost = Required>; +export declare type DependencyResolverHost = Required> & { + useSourceOfProjectReferenceRedirect?(): boolean; +}; export declare class DependencyResolverFactory { create(host: DependencyResolverHost, program: ts.Program): DependencyResolver; } diff --git a/baselines/packages/ymir/api/src/index.d.ts b/baselines/packages/ymir/api/src/index.d.ts index d300f7fbb..3e080ddb0 100644 --- a/baselines/packages/ymir/api/src/index.d.ts +++ b/baselines/packages/ymir/api/src/index.d.ts @@ -352,7 +352,9 @@ export interface RawLineSwitchRule { } export interface FileFilterContext { program: ts.Program; - host: Required>; + host: Required> & { + useSourceOfProjectReferenceRedirect?(): boolean; + }; } export interface FileFilterFactory { create(context: FileFilterContext): FileFilter; diff --git a/packages/wotan/src/project-host.ts b/packages/wotan/src/project-host.ts index 88656457d..909e09bfb 100644 --- a/packages/wotan/src/project-host.ts +++ b/packages/wotan/src/project-host.ts @@ -291,10 +291,16 @@ export class ProjectHost implements ts.CompilerHost { ); } - public resolveModuleNames(names: string[], file: string, _?: unknown, reference?: ts.ResolvedProjectReference) { + public resolveModuleNames( + names: string[], + file: string, + _: unknown | undefined, + reference: ts.ResolvedProjectReference | undefined, + options: ts.CompilerOptions, + ) { const seen = new Map(); const resolve = (name: string) => - ts.resolveModuleName(name, file, this.compilerOptions, this, this.moduleResolutionCache, reference).resolvedModule; + ts.resolveModuleName(name, file, options, this, this.moduleResolutionCache, reference).resolvedModule; return names.map((name) => resolveCachedResult(seen, name, resolve)); } } diff --git a/packages/wotan/src/services/default/file-filter.ts b/packages/wotan/src/services/default/file-filter.ts index 995e32a1c..177af9ac1 100644 --- a/packages/wotan/src/services/default/file-filter.ts +++ b/packages/wotan/src/services/default/file-filter.ts @@ -17,6 +17,8 @@ class DefaultFileFilter implements FileFilter { private libDirectory = unixifyPath(path.dirname(ts.getDefaultLibFilePath(this.options))) + '/'; private typeRoots: ReadonlyArray | undefined = undefined; private outputsOfReferencedProjects: ReadonlyArray | undefined = undefined; + private useSourceOfProjectReferenceRedirect = this.host.useSourceOfProjectReferenceRedirect?.() === true && + !this.options.disableSourceOfProjectReferenceRedirect; constructor(private program: ts.Program, private host: FileFilterContext['host']) {} @@ -28,6 +30,8 @@ class DefaultFileFilter implements FileFilter { return this.rootNames.includes(fileName); if (this.program.isSourceFileFromExternalLibrary(file)) return false; + if (this.useSourceOfProjectReferenceRedirect && this.isSourceFileOfProjectReference(fileName)) + return false; if (!fileName.endsWith('.d.ts')) return true; if ( @@ -56,4 +60,11 @@ class DefaultFileFilter implements FileFilter { ); return this.outputsOfReferencedProjects.includes(fileName); } + + private isSourceFileOfProjectReference(fileName: string) { + for (const ref of iterateProjectReferences(this.program.getResolvedProjectReferences())) + if (ref.commandLine.fileNames.includes(fileName)) + return true; + return false; + } } diff --git a/packages/wotan/src/services/dependency-resolver.ts b/packages/wotan/src/services/dependency-resolver.ts index 7dab392b2..05df7113f 100644 --- a/packages/wotan/src/services/dependency-resolver.ts +++ b/packages/wotan/src/services/dependency-resolver.ts @@ -1,8 +1,9 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; import { isModuleDeclaration, isNamespaceExportDeclaration, findImports, ImportKind } from 'tsutils'; -import { resolveCachedResult, getOutputFileNamesOfProjectReference, iterateProjectReferences } from '../utils'; +import { resolveCachedResult, getOutputFileNamesOfProjectReference, iterateProjectReferences, unixifyPath } from '../utils'; import bind from 'bind-decorator'; +import * as path from 'path'; export interface DependencyResolver { update(program: ts.Program, updatedFile: string): void; @@ -10,7 +11,9 @@ export interface DependencyResolver { getFilesAffectingGlobalScope(): readonly string[]; } -export type DependencyResolverHost = Required>; +export type DependencyResolverHost = Required> & { + useSourceOfProjectReferenceRedirect?(): boolean; +}; @injectable() export class DependencyResolverFactory { @@ -31,6 +34,8 @@ class DependencyResolverImpl implements DependencyResolver { private fileToProjectReference: ReadonlyMap | undefined = undefined; private fileMetadata = new Map(); private compilerOptions = this.program.getCompilerOptions(); + private useSourceOfProjectReferenceRedirect = this.host.useSourceOfProjectReferenceRedirect?.() === true && + !this.compilerOptions.disableSourceOfProjectReferenceRedirect; private state: DependencyResolverState | undefined = undefined; @@ -73,8 +78,8 @@ class DependencyResolverImpl implements DependencyResolver { for (const file of files) { const resolved = this.getExternalReferences(file).get(module); // if an augmentation's identifier can be resolved from the declaring file, the augmentation applies to the resolved path - if (resolved !== null) { - addToList(moduleAugmentations, resolved!, file); + if (resolved != null) { // tslint:disable-line:triple-equals + addToList(moduleAugmentations, resolved, file); } else { // if a pattern ambient module matches the augmented identifier, the augmentation applies to that const matchingPattern = getBestMatchingPattern(module, patternAmbientModules.keys()); @@ -144,8 +149,6 @@ class DependencyResolverImpl implements DependencyResolver { @bind private collectExternalReferences(fileName: string): Map { - // TODO useSourceOfProjectReferenceRedirect - // TODO referenced files // TODO add tslib if importHelpers is enabled const sourceFile = this.program.getSourceFile(fileName)!; const references = new Set(findImports(sourceFile, ImportKind.All, false).map(({text}) => text)); @@ -157,9 +160,26 @@ class DependencyResolverImpl implements DependencyResolver { return result; this.fileToProjectReference ??= createProjectReferenceMap(this.program.getResolvedProjectReferences()); const arr = Array.from(references); - const resolved = this.host.resolveModuleNames(arr, fileName, undefined, this.fileToProjectReference.get(fileName), this.compilerOptions); - for (let i = 0; i < resolved.length; ++i) - result.set(arr[i], resolved[i]?.resolvedFileName ?? null); + const resolved = + this.host.resolveModuleNames(arr, fileName, undefined, this.fileToProjectReference.get(fileName), this.compilerOptions); + for (let i = 0; i < resolved.length; ++i) { + const current = resolved[i]; + if (current === undefined) { + result.set(arr[i], null); + } else { + const projectReference = this.useSourceOfProjectReferenceRedirect + ? this.fileToProjectReference.get(current.resolvedFileName) + : undefined; + if (projectReference === undefined) { + result.set(arr[i], current.resolvedFileName); + } else if (projectReference.commandLine.options.outFile) { + // with outFile the files must be global anyway, so we don't care about the exact file + result.set(arr[i], projectReference.commandLine.fileNames[0]); + } else { + result.set(arr[i], getSourceOfProjectReferenceRedirect(current.resolvedFileName, projectReference)); + } + } + } return result; } } @@ -224,3 +244,19 @@ function collectFileMetadata(sourceFile: ts.SourceFile): MetaData { } return {ambientModules, isExternalModule, affectsGlobalScope: affectsGlobalScope === true}; } + +function getSourceOfProjectReferenceRedirect(outputFileName: string, ref: ts.ResolvedProjectReference): string { + const options = ref.commandLine.options; + const projectDirectory = path.dirname(ref.sourceFile.fileName); + const origin = unixifyPath(path.resolve( + options.rootDir || projectDirectory, + path.relative(options.declarationDir || options.outDir || projectDirectory, outputFileName.slice(0, -5)), + )); + + for (const extension of ['.ts', '.tsx', '.js', '.jsx']) { + const name = origin + extension; + if (ref.commandLine.fileNames.includes(name)) + return name; + } + return outputFileName; // should never happen +} diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 2bbf61915..425491682 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -1,7 +1,7 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; import { DependencyResolver, DependencyResolverFactory, DependencyResolverHost } from './dependency-resolver'; -import { resolveCachedResult, djb2 } from '../utils'; +import { resolveCachedResult, djb2, unixifyPath } from '../utils'; import bind from 'bind-decorator'; import { EffectiveConfiguration, Finding, ReducedConfiguration, StatePersistence, StaticProgramState } from '@fimbul/ymir'; import debug = require('debug'); @@ -117,7 +117,7 @@ class ProgramStateImpl implements ProgramState { @bind private makeRelativePath(fileName: string) { - return path.posix.relative(this.canonicalProjectDirectory, this.host.getCanonicalFileName(fileName)); + return unixifyPath(path.relative(this.canonicalProjectDirectory, this.host.getCanonicalFileName(fileName))); } public getUpToDateResult(fileName: string, config: ReducedConfiguration) { diff --git a/packages/ymir/src/index.ts b/packages/ymir/src/index.ts index 2229ada76..88264a5d5 100644 --- a/packages/ymir/src/index.ts +++ b/packages/ymir/src/index.ts @@ -503,7 +503,9 @@ export interface RawLineSwitchRule { export interface FileFilterContext { program: ts.Program; - host: Required>; + host: Required> & { + useSourceOfProjectReferenceRedirect?(): boolean; + }; } export interface FileFilterFactory { From 15b090a62a69bd3462c34324f7063f52152f33c3 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sun, 31 Jan 2021 13:28:08 +0100 Subject: [PATCH 10/37] small refactoring --- packages/wotan/src/services/program-state.ts | 133 ++++++++++--------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 425491682..74a0a921b 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -43,7 +43,7 @@ const STATE_VERSION = 1; const oldStateSymbol = Symbol('oldState'); class ProgramStateImpl implements ProgramState { - private projectDirectory = path.posix.dirname(this.project); + private projectDirectory = unixifyPath(path.dirname(this.project)); private canonicalProjectDirectory = this.host.getCanonicalFileName(this.projectDirectory); private optionsHash = computeCompilerOptionsHash(this.program.getCompilerOptions(), this.projectDirectory); private assumeChangesOnlyAffectDirectDependencies = @@ -188,79 +188,82 @@ class ProgramStateImpl implements ProgramState { while (true) { index = indexQueue.pop()!; fileName = fileNameQueue.pop()!; - switch (this.dependenciesUpToDate[index]) { - case DependencyState.Outdated: - return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - case DependencyState.Unknown: { - let earliestCircularDependency = Number.MAX_SAFE_INTEGER; - let childCount = 0; - - for (const cycle of cycles) { - if (cycle.has(index)) { - // we already know this is a circular dependency, skip this one and simply mark the parent as circular - earliestCircularDependency = - findCircularDependencyOfCycle(parents, circularDependenciesQueue, cycle); - break; - } + processFile: { + switch (this.dependenciesUpToDate[index]) { + case DependencyState.Outdated: + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + case DependencyState.Ok: + break processFile; + } + for (const cycle of cycles) { + if (cycle.has(index)) { + // we already know this is a circular dependency, skip this one and simply mark the parent as circular + setCircularDependency( + parents, + circularDependenciesQueue, + index, + cycles, + findCircularDependencyOfCycle(parents, circularDependenciesQueue, cycle), + ); + break processFile; } - if (earliestCircularDependency === Number.MAX_SAFE_INTEGER) { - const old = oldState.files[index]; - const dependencies = this.resolver.getDependencies(fileName); - const keys = Object.keys(old.dependencies); + } + let earliestCircularDependency = Number.MAX_SAFE_INTEGER; + let childCount = 0; + const old = oldState.files[index]; + const dependencies = this.resolver.getDependencies(fileName); + const keys = Object.keys(old.dependencies); - if (dependencies.size !== keys.length) + if (dependencies.size !== keys.length) + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + for (const key of keys) { + let newDeps = dependencies.get(key); + if (newDeps === undefined) + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + const oldDeps = old.dependencies[key]; + if (oldDeps === null) { + if (newDeps !== null) return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - for (const key of keys) { - let newDeps = dependencies.get(key); - if (newDeps === undefined) - return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - const oldDeps = old.dependencies[key]; - if (oldDeps === null) { - if (newDeps !== null) - return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - continue; - } - if (newDeps === null) - return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - newDeps = Array.from(new Set(newDeps)); - if (newDeps.length !== oldDeps.length) - return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - const newDepsWithHash = this.sortByHash(newDeps); - for (let i = 0; i < newDepsWithHash.length; ++i) { - const oldDepState = oldState.files[oldDeps[i]]; - if (newDepsWithHash[i].hash !== oldDepState.hash) - return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - if (!this.assumeChangesOnlyAffectDirectDependencies && fileName !== newDepsWithHash[i].fileName) { - const indexInQueue = parents.indexOf(oldDeps[i]); - if (indexInQueue === -1) { - // no circular dependency - fileNameQueue.push(newDepsWithHash[i].fileName); - indexQueue.push(oldDeps[i]); - ++childCount; - } else if (indexInQueue < earliestCircularDependency) { - earliestCircularDependency = indexInQueue; - } - } + continue; + } + if (newDeps === null) + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + newDeps = Array.from(new Set(newDeps)); + if (newDeps.length !== oldDeps.length) + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + const newDepsWithHash = this.sortByHash(newDeps); + for (let i = 0; i < newDepsWithHash.length; ++i) { + const oldDepState = oldState.files[oldDeps[i]]; + if (newDepsWithHash[i].hash !== oldDepState.hash) + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + if (!this.assumeChangesOnlyAffectDirectDependencies && fileName !== newDepsWithHash[i].fileName) { + const indexInQueue = parents.indexOf(oldDeps[i]); + if (indexInQueue === -1) { + // no circular dependency + fileNameQueue.push(newDepsWithHash[i].fileName); + indexQueue.push(oldDeps[i]); + ++childCount; + } else if (indexInQueue < earliestCircularDependency) { + earliestCircularDependency = indexInQueue; } } } + } - if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) { - earliestCircularDependency = - setCircularDependency(parents, circularDependenciesQueue, index, cycles, earliestCircularDependency); - } else if (childCount === 0) { - this.dependenciesUpToDate[index] = DependencyState.Ok; - } - if (childCount !== 0) { - parents.push(index); - childCounts.push(childCount); - circularDependenciesQueue.push(earliestCircularDependency); - continue; - } + if (earliestCircularDependency !== Number.MAX_SAFE_INTEGER) { + earliestCircularDependency = + setCircularDependency(parents, circularDependenciesQueue, index, cycles, earliestCircularDependency); + } else if (childCount === 0) { + this.dependenciesUpToDate[index] = DependencyState.Ok; + } + if (childCount !== 0) { + parents.push(index); + childCounts.push(childCount); + circularDependenciesQueue.push(earliestCircularDependency); + continue; } } // we only get here for files with no children to process - if (parents.length === 0) return true; // only happens if the initial file has no dependencies or they are all already known as Ok @@ -541,7 +544,7 @@ function computeCompilerOptionsHash(options: ts.CompilerOptions, relativeTo: str return '' + djb2(JSON.stringify(obj)); function makeRelativePath(p: string) { - return path.posix.relative(relativeTo, p); + return unixifyPath(path.relative(relativeTo, p)); } } From 2bb629045bd01a96f4cd24d32437868ce9d5be51 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 16:33:31 +0100 Subject: [PATCH 11/37] cannot use --cache without --project --- packages/wotan/src/argparse.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/wotan/src/argparse.ts b/packages/wotan/src/argparse.ts index dc65ec02e..c56d426aa 100644 --- a/packages/wotan/src/argparse.ts +++ b/packages/wotan/src/argparse.ts @@ -163,13 +163,16 @@ function parseLintCommand( } } + const usesProject = result.project.length !== 0 || result.files.length === 0; if (result.extensions !== undefined) { if (result.extensions.length === 0) { result.extensions = undefined; - } else if (result.project.length !== 0 || result.files.length === 0) { + } else if (usesProject) { throw new ConfigurationError("Options '--ext' and '--project' cannot be used together."); } } + if (result.cache && !usesProject) + throw new ConfigurationError("Option '--cache' can only be used together with '--project'"); return result; } From d3ad02ba03b2cb9c19a661b4939b5e45ef05c6f8 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 16:41:54 +0100 Subject: [PATCH 12/37] revert unnecessary changes to file-filter --- baselines/packages/ymir/api/src/index.d.ts | 4 +--- packages/wotan/src/services/default/file-filter.ts | 11 ----------- packages/ymir/src/index.ts | 4 +--- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/baselines/packages/ymir/api/src/index.d.ts b/baselines/packages/ymir/api/src/index.d.ts index 3e080ddb0..d300f7fbb 100644 --- a/baselines/packages/ymir/api/src/index.d.ts +++ b/baselines/packages/ymir/api/src/index.d.ts @@ -352,9 +352,7 @@ export interface RawLineSwitchRule { } export interface FileFilterContext { program: ts.Program; - host: Required> & { - useSourceOfProjectReferenceRedirect?(): boolean; - }; + host: Required>; } export interface FileFilterFactory { create(context: FileFilterContext): FileFilter; diff --git a/packages/wotan/src/services/default/file-filter.ts b/packages/wotan/src/services/default/file-filter.ts index 177af9ac1..995e32a1c 100644 --- a/packages/wotan/src/services/default/file-filter.ts +++ b/packages/wotan/src/services/default/file-filter.ts @@ -17,8 +17,6 @@ class DefaultFileFilter implements FileFilter { private libDirectory = unixifyPath(path.dirname(ts.getDefaultLibFilePath(this.options))) + '/'; private typeRoots: ReadonlyArray | undefined = undefined; private outputsOfReferencedProjects: ReadonlyArray | undefined = undefined; - private useSourceOfProjectReferenceRedirect = this.host.useSourceOfProjectReferenceRedirect?.() === true && - !this.options.disableSourceOfProjectReferenceRedirect; constructor(private program: ts.Program, private host: FileFilterContext['host']) {} @@ -30,8 +28,6 @@ class DefaultFileFilter implements FileFilter { return this.rootNames.includes(fileName); if (this.program.isSourceFileFromExternalLibrary(file)) return false; - if (this.useSourceOfProjectReferenceRedirect && this.isSourceFileOfProjectReference(fileName)) - return false; if (!fileName.endsWith('.d.ts')) return true; if ( @@ -60,11 +56,4 @@ class DefaultFileFilter implements FileFilter { ); return this.outputsOfReferencedProjects.includes(fileName); } - - private isSourceFileOfProjectReference(fileName: string) { - for (const ref of iterateProjectReferences(this.program.getResolvedProjectReferences())) - if (ref.commandLine.fileNames.includes(fileName)) - return true; - return false; - } } diff --git a/packages/ymir/src/index.ts b/packages/ymir/src/index.ts index 88264a5d5..2229ada76 100644 --- a/packages/ymir/src/index.ts +++ b/packages/ymir/src/index.ts @@ -503,9 +503,7 @@ export interface RawLineSwitchRule { export interface FileFilterContext { program: ts.Program; - host: Required> & { - useSourceOfProjectReferenceRedirect?(): boolean; - }; + host: Required>; } export interface FileFilterFactory { From 00ce2b5c0015fc694f74f10b586f0fe03b44a028 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 17:21:57 +0100 Subject: [PATCH 13/37] Add tests for StatePersistence --- packages/wotan/test/services.spec.ts | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/packages/wotan/test/services.spec.ts b/packages/wotan/test/services.spec.ts index 79432a874..fe10816d9 100644 --- a/packages/wotan/test/services.spec.ts +++ b/packages/wotan/test/services.spec.ts @@ -11,6 +11,8 @@ import { DirectoryService, ConfigurationError, BuiltinResolver, + StatePersistence, + StaticProgramState, } from '@fimbul/ymir'; import { NodeDirectoryService } from '../src/services/default/directory-service'; import * as os from 'os'; @@ -31,6 +33,8 @@ import { DefaultDeprecationHandler } from '../src/services/default/deprecation-h import { FormatterLoader } from '../src/services/formatter-loader'; import { ProcessorLoader } from '../src/services/processor-loader'; import { satisfies } from 'semver'; +import * as yaml from 'js-yaml'; +import { DefaultStatePersistence } from '../src/services/default/state-persistence'; test('CacheFactory', (t) => { const cm = new DefaultCacheFactory(); @@ -427,3 +431,72 @@ test('ProcessorLoader', (t) => { r = require; t.throws(() => loader.loadProcessor('./fooBarBaz'), {message: /^Cannot find module '\.\/fooBarBaz'$/m}); }); + +test('StatePersistence', (t) => { + const container = new Container(); + container.bind(CachedFileSystem).toSelf(); + container.bind(CacheFactory).to(DefaultCacheFactory); + container.bind(StatePersistence).to(DefaultStatePersistence); + const state: StaticProgramState = { + cs: true, + files: [], + global: [], + lookup: {}, + options: '', + ts: ts.version, + v: 1, + }; + const fileContent = {state, v: 1}; + let written = false; + container.bind(FileSystem).toConstantValue({ + createDirectory() { throw new Error(); }, + deleteFile() { throw new Error(); }, + readDirectory() { throw new Error(); }, + realpath() { throw new Error(); }, + normalizePath: NodeFileSystem.normalizePath, + stat(f) { + switch (f) { + case './tsconfig-nonexist.fimbullintercache': + return { isDirectory() { return true; }, isFile() { return false; } }; + case './tsconfig-mismatch.fimbullintercache': + case './tsconfig-throws.fimbullintercache': + case './tsconfig-correct.fimbullintercache': + return { isDirectory() { return false; }, isFile() { return true; } }; + default: + throw t.fail('unexpected file ' + f); + } + }, + readFile(f) { + switch (f) { + case './tsconfig-correct.fimbullintercache': + return yaml.dump(fileContent); + case './tsconfig-mismatch.fimbullintercache': + return yaml.dump({state, v: 0}); + case './tsconfig-throws.fimbullintercache': + throw new Error(); + default: + throw t.fail('unexpected file ' + f); + } + }, + writeFile(f, content) { + if (f === './tsconfig-throws.fimbullintercache') + throw new Error(); + t.is(f, './tsconfig-correct.fimbullintercache'); + t.deepEqual(yaml.load(content), fileContent); + written = true; + }, + }); + + const service = container.get(StatePersistence); + + t.is(service.loadState('./tsconfig-nonexist.json'), undefined); + t.is(service.loadState('./tsconfig-mismatch.json'), undefined); + t.is(service.loadState('./tsconfig-throws.json'), undefined); + t.deepEqual(service.loadState('./tsconfig-correct.json'), state); + + service.saveState('./tsconfig-throws.json', state); + t.is(written, false); + + service.saveState('./tsconfig-correct.json', state); + t.is(written, true); +}); From 42e5a6380de69df3ddfe0823ee56b630d3237650 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 19:10:50 +0100 Subject: [PATCH 14/37] add test for argparse --- packages/wotan/src/services/program-state.ts | 1 + packages/wotan/test/argparse.spec.ts | 80 ++++++++++++++++++++ packages/wotan/test/services.spec.ts | 4 + 3 files changed, 85 insertions(+) diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 74a0a921b..97d7224f7 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -362,6 +362,7 @@ function findCircularDependencyOfCycle(parents: readonly number[], circularDepen if (dep !== Number.MAX_SAFE_INTEGER && cycle.has(parents[i])) return dep; } + /* istanbul ignore next */ throw new Error('should never happen'); } diff --git a/packages/wotan/test/argparse.spec.ts b/packages/wotan/test/argparse.spec.ts index d96e0ebba..9bda27a51 100644 --- a/packages/wotan/test/argparse.spec.ts +++ b/packages/wotan/test/argparse.spec.ts @@ -100,6 +100,9 @@ test('parseGlobalOptions', (t) => { t.is(parseGlobalOptions({reportUselessDirectives: 'off'}).reportUselessDirectives, false); t.is(parseGlobalOptions({reportUselessDirectives: true}).reportUselessDirectives, true); t.is(parseGlobalOptions({reportUselessDirectives: false}).reportUselessDirectives, false); + + t.is(parseGlobalOptions({cache: false}).cache, false); + t.is(parseGlobalOptions({cache: true}).cache, true); }); test('defaults to lint command', (t) => { @@ -717,6 +720,82 @@ test('parses lint command', (t) => { 'only parses severity or boolean as value for --report-useless-directives', ); + t.deepEqual( + parseArguments(['lint', '--cache', '-p', '.']), + { + command: CommandName.Lint, + modules: [], + config: undefined, + files: [], + exclude: [], + formatter: undefined, + project: ['.'], + references: false, + fix: false, + extensions: undefined, + reportUselessDirectives: false, + cache: true, + }, + 'parses --cache with --project', + ); + + t.deepEqual( + parseArguments(['lint', '--cache', 'true', '-p', '.']), + { + command: CommandName.Lint, + modules: [], + config: undefined, + files: [], + exclude: [], + formatter: undefined, + project: ['.'], + references: false, + fix: false, + extensions: undefined, + reportUselessDirectives: false, + cache: true, + }, + 'parses --cache with --project', + ); + + t.deepEqual( + parseArguments(['lint', '--cache', 'false', '-p', '.']), + { + command: CommandName.Lint, + modules: [], + config: undefined, + files: [], + exclude: [], + formatter: undefined, + project: ['.'], + references: false, + fix: false, + extensions: undefined, + reportUselessDirectives: false, + cache: false, + }, + 'parses --cache with --project', + ); + + t.deepEqual( + parseArguments(['lint', '--cache']), + { + command: CommandName.Lint, + modules: [], + config: undefined, + files: [], + exclude: [], + formatter: undefined, + project: ['.'], + references: false, + fix: false, + extensions: undefined, + reportUselessDirectives: false, + cache: false, + }, + 'parses --cache with implicit --project', + ); + t.throws(() => parseArguments(['lint', '--foobar']), { message: "Unknown option '--foobar'." }); t.throws(() => parseArguments(['lint', '-m']), { message: "Option '-m' expects an argument." }); @@ -727,6 +806,7 @@ test('parses lint command', (t) => { t.throws(() => parseArguments(['lint', '--ext']), { message: "Option '--ext' expects an argument." }); t.throws(() => parseArguments(['lint', '--ext', 'mjs']), { message: "Options '--ext' and '--project' cannot be used together." }); t.throws(() => parseArguments(['lint', '--ext', 'mjs', '-p', '.']), { message: "Options '--ext' and '--project' cannot be used together." }); + t.throws(() => parseArguments(['lint', '--cache', 'a.ts']), { message: "Option '--cache' can only be used together with '--project'" }); }); test('parses save command', (t) => { diff --git a/packages/wotan/test/services.spec.ts b/packages/wotan/test/services.spec.ts index fe10816d9..8df7ae4b1 100644 --- a/packages/wotan/test/services.spec.ts +++ b/packages/wotan/test/services.spec.ts @@ -460,6 +460,7 @@ test('StatePersistence', (t) => { return { isDirectory() { return true; }, isFile() { return false; } }; case './tsconfig-mismatch.fimbullintercache': case './tsconfig-throws.fimbullintercache': + case './tsconfig-empty.fimbullintercache': case './tsconfig-correct.fimbullintercache': return { isDirectory() { return false; }, isFile() { return true; } }; default: @@ -472,6 +473,8 @@ test('StatePersistence', (t) => { return yaml.dump(fileContent); case './tsconfig-mismatch.fimbullintercache': return yaml.dump({state, v: 0}); + case './tsconfig-empty.fimbullintercache': + return ''; case './tsconfig-throws.fimbullintercache': throw new Error(); default: @@ -492,6 +495,7 @@ test('StatePersistence', (t) => { t.is(service.loadState('./tsconfig-nonexist.json'), undefined); t.is(service.loadState('./tsconfig-mismatch.json'), undefined); t.is(service.loadState('./tsconfig-throws.json'), undefined); + t.is(service.loadState('./tsconfig-empty.json'), undefined); t.deepEqual(service.loadState('./tsconfig-correct.json'), state); service.saveState('./tsconfig-throws.json', state); From f2fbb4b2635101287063fcb1733b10386ae7395b Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 20:46:39 +0100 Subject: [PATCH 15/37] move config hashing to runner --- docs/api.md | 5 ++- packages/wotan/src/runner.ts | 35 +++++++++++++++--- packages/wotan/src/services/program-state.ts | 38 +++++--------------- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/docs/api.md b/docs/api.md index 26f89672f..d619a2ad4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -9,11 +9,13 @@ There are several core services that are provided by Wotan through the Container * `CachedFileSystem` is a wrapper for the low level `FileSystem` service, which caches the file system layout. File contents are not cached. * `ConfigurationManager` is the place for everything related to configuration handling. Internally it uses `ConfigurationProvider` to find, load and parse configuration files. Parsed configuration files are cached. +* `DependencyResolverFactory` creates a service to determine how files in the program affect each other. * `FormatterLoader` loads core and custom formatters via `FormatterLoaderHost`. * `Linter` executes a given set of rules on a SourceFile. It automatically loads enabled rules using `RuleLoader` and filters out disabled findings using `FindingFilterFactory`. `Linter` can also automatically fix findings and return the fixed source code. It does not access the file system. * `ProcessorLoader` loads and caches processors using `Resolver`. +* `ProgramStateFactory` creates a service to get lint results for up-to-date files from cache and update the cache as necessary. Uses `StatePersistence` to load the cache for the current project. Uses `DependencyResolverFactory` to find out about file dependencies. * `RuleLoader` loads and caches core and custom rules via `RuleLoaderHost`. -* `Runner` is used to lint a collection of files. If you want to lint a project, you provide the path of one or more `tsconfig.json` and it creates the project internally. `Runner` loads the source code from the file system, loads configuration from `ConfigurationManager`, applies processors if specified in the configuration and lints all (matching) files using `Linter`. It uses `FileFilterFactory` to filter out non-user code. +* `Runner` is used to lint a collection of files. If you want to lint a project, you provide the path of one or more `tsconfig.json` and it creates the project internally. `Runner` loads the source code from the file system, loads configuration from `ConfigurationManager`, applies processors if specified in the configuration and lints all (matching) files using `Linter`. It uses `FileFilterFactory` to filter out non-user code. If caching is enabled, it uses `ProgramStateFactory` to load the cached results and update the cache. These core services use other abstractions for the low level tasks. That enables you to change the behavior of certain services without the need to implement the whole thing. The default implementations (targeting the Node.js runtime environment) are provided throug the ContainerModule `DEFAULT_DI_MODULE`. The default implementation is only used if there is no binding for the identifier. @@ -31,6 +33,7 @@ The default implementations (targeting the Node.js runtime environment) are prov * `MessageHandler` is used for user facing messages. `log` is called for the result of a command, `warn` is called everytime a warning event occurs and `error` is used to display exception messages. * `Resolver` (`NodeResolver`) is an abstraction for `require()` and `require.resolve()`. It's used to locate and load external resources (configuration, scripts, ...). * `RuleLoaderHost` (`NodeRuleLoader`) is used to resolve and require a rule. +* `StatePersistence` (`DefaultStatePersistence`) is responsible to load and save the cache for a given `tsconfig.json`. ## Example diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index 709ff98c5..b121183ad 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -9,11 +9,13 @@ import { MessageHandler, FileFilterFactory, Severity, + ReducedConfiguration, + EffectiveConfiguration, } from '@fimbul/ymir'; import * as path from 'path'; import * as ts from 'typescript'; import * as glob from 'glob'; -import { unixifyPath, hasSupportedExtension, addUnique, flatMap, hasParseErrors, invertChangeRange } from './utils'; +import { unixifyPath, hasSupportedExtension, addUnique, flatMap, hasParseErrors, invertChangeRange, djb2 } from './utils'; import { Minimatch, IMinimatch } from 'minimatch'; import { ProcessorLoader } from './services/processor-loader'; import { injectable } from 'inversify'; @@ -118,8 +120,8 @@ export class Runner { const originalContent = mapped === undefined ? sourceFile.text : mapped.originalContent; let summary: FileSummary; const fix = shouldFix(sourceFile, options, originalName); - // TODO consider reportUselessDirectives in cache - const resultFromCache = programState?.getUpToDateResult(sourceFile.fileName, effectiveConfig); + const configHash = programState === undefined ? undefined : createConfigHash(effectiveConfig, linterOptions); + const resultFromCache = programState?.getUpToDateResult(sourceFile.fileName, configHash!); if (fix) { let updatedFile = false; summary = this.linter.lintAndFix( @@ -164,7 +166,7 @@ export class Runner { }; } if (programState !== undefined && resultFromCache !== summary.findings) - programState.setFileResult(file, effectiveConfig, summary.findings); + programState.setFileResult(file, configHash!, summary.findings); yield [originalName, summary]; } programState?.save(); @@ -446,6 +448,31 @@ function shouldFix(sourceFile: ts.SourceFile, options: Pick, return options.fix; } + +function createConfigHash(config: ReducedConfiguration, linterOptions: LinterOptions) { + return '' + djb2(JSON.stringify({ + rules: mapToObject(config.rules, stripRuleConfig), + settings: mapToObject(config.settings, identity), + reportUselessDirectives: linterOptions.reportUselessDirectives, + })); +} + +function mapToObject(map: ReadonlyMap, transform: (v: T) => U) { + const result: Record = {}; + for (const [key, value] of map) + result[key] = transform(value); + return result; +} + +function identity(v: T) { + return v; +} + +function stripRuleConfig({rulesDirectories: _ignored, ...rest}: EffectiveConfiguration.RuleConfig) { + return rest; +} + + declare module 'typescript' { function matchFiles( path: string, diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 97d7224f7..34a8fb829 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -3,7 +3,7 @@ import * as ts from 'typescript'; import { DependencyResolver, DependencyResolverFactory, DependencyResolverHost } from './dependency-resolver'; import { resolveCachedResult, djb2, unixifyPath } from '../utils'; import bind from 'bind-decorator'; -import { EffectiveConfiguration, Finding, ReducedConfiguration, StatePersistence, StaticProgramState } from '@fimbul/ymir'; +import { Finding, StatePersistence, StaticProgramState } from '@fimbul/ymir'; import debug = require('debug'); import { isCompilerOptionEnabled } from 'tsutils'; import * as path from 'path'; @@ -12,8 +12,8 @@ const log = debug('wotan:programState'); export interface ProgramState { update(program: ts.Program, updatedFile: string): void; - getUpToDateResult(fileName: string, config: EffectiveConfiguration): readonly Finding[] | undefined; - setFileResult(fileName: string, config: EffectiveConfiguration, result: readonly Finding[]): void; + getUpToDateResult(fileName: string, configHash: string): readonly Finding[] | undefined; + setFileResult(fileName: string, configHash: string, result: readonly Finding[]): void; save(): void; } @@ -120,7 +120,7 @@ class ProgramStateImpl implements ProgramState { return unixifyPath(path.relative(this.canonicalProjectDirectory, this.host.getCanonicalFileName(fileName))); } - public getUpToDateResult(fileName: string, config: ReducedConfiguration) { + public getUpToDateResult(fileName: string, configHash: string) { const oldState = this.tryReuseOldState(); if (oldState === undefined) return; @@ -130,7 +130,7 @@ class ProgramStateImpl implements ProgramState { const old = oldState.files[index]; if ( old.result === undefined || - old.config !== '' + djb2(JSON.stringify(stripConfig(config))) || + old.config !== configHash || old.hash !== this.getFileHash(fileName) || !this.fileDependenciesUpToDate(fileName, index, oldState) ) @@ -139,7 +139,7 @@ class ProgramStateImpl implements ProgramState { return old.result; } - public setFileResult(fileName: string, config: ReducedConfiguration, result: ReadonlyArray) { + public setFileResult(fileName: string, configHash: string, result: ReadonlyArray) { if (!this.isFileUpToDate(fileName)) { log('File %s is outdated, merging current state into old state', fileName); // we need to create a state where the file is up-to-date @@ -150,7 +150,7 @@ class ProgramStateImpl implements ProgramState { this.fileResults = new Map(); this.dependenciesUpToDate = new Uint8Array(newState.files.length).fill(DependencyState.Ok); } - this.fileResults.set(fileName, {result, config: '' + djb2(JSON.stringify(stripConfig(config)))}); + this.fileResults.set(fileName, {result, config: configHash}); } private isFileUpToDate(fileName: string): boolean { @@ -170,7 +170,7 @@ class ProgramStateImpl implements ProgramState { } } - private fileDependenciesUpToDate(fileName: string, index: number, oldState: StaticProgramState): boolean { // TODO something is wrong + private fileDependenciesUpToDate(fileName: string, index: number, oldState: StaticProgramState): boolean { // File names that are waiting to be processed, each iteration of the loop processes one file const fileNameQueue = [fileName]; // For each entry in `fileNameQueue` this holds the index of that file in `oldState.files` @@ -548,25 +548,3 @@ function computeCompilerOptionsHash(options: ts.CompilerOptions, relativeTo: str return unixifyPath(path.relative(relativeTo, p)); } } - -function stripConfig(config: ReducedConfiguration) { - return { - rules: mapToObject(config.rules, stripRule), - settings: mapToObject(config.settings, identity), - }; -} - -function mapToObject(map: ReadonlyMap, transform: (v: T) => U) { - const result: Record = {}; - for (const [key, value] of map) - result[key] = transform(value); - return result; -} - -function identity(v: T) { - return v; -} - -function stripRule({rulesDirectories: _ignored, ...rest}: EffectiveConfiguration.RuleConfig) { - return rest; -} From bce5c9a469b6d6896bee6e5a3ccc0dd64ddc3520 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 21:05:20 +0100 Subject: [PATCH 16/37] don't include empty dependenies object in cache --- .../packages/wotan/api/src/services/program-state.d.ts | 10 +++++----- baselines/packages/ymir/api/src/index.d.ts | 2 +- packages/wotan/src/runner.ts | 2 +- packages/wotan/src/services/program-state.ts | 9 ++++++++- packages/ymir/src/index.ts | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/baselines/packages/wotan/api/src/services/program-state.d.ts b/baselines/packages/wotan/api/src/services/program-state.d.ts index 9fc97aab6..7510a7b82 100644 --- a/baselines/packages/wotan/api/src/services/program-state.d.ts +++ b/baselines/packages/wotan/api/src/services/program-state.d.ts @@ -1,10 +1,10 @@ import * as ts from 'typescript'; import { DependencyResolver, DependencyResolverFactory, DependencyResolverHost } from './dependency-resolver'; -import { EffectiveConfiguration, Finding, ReducedConfiguration, StatePersistence } from '@fimbul/ymir'; +import { Finding, StatePersistence } from '@fimbul/ymir'; export interface ProgramState { update(program: ts.Program, updatedFile: string): void; - getUpToDateResult(fileName: string, config: EffectiveConfiguration): readonly Finding[] | undefined; - setFileResult(fileName: string, config: EffectiveConfiguration, result: readonly Finding[]): void; + getUpToDateResult(fileName: string, configHash: string): readonly Finding[] | undefined; + setFileResult(fileName: string, configHash: string, result: readonly Finding[]): void; save(): void; } export declare class ProgramStateFactory { @@ -16,8 +16,8 @@ declare const oldStateSymbol: unique symbol; declare class ProgramStateImpl implements ProgramState { constructor(host: ProgramStateHost, program: ts.Program, resolver: DependencyResolver, statePersistence: StatePersistence, project: string); update(program: ts.Program, updatedFile: string): void; - getUpToDateResult(fileName: string, config: ReducedConfiguration): readonly Finding[] | undefined; - setFileResult(fileName: string, config: ReducedConfiguration, result: ReadonlyArray): void; + getUpToDateResult(fileName: string, configHash: string): readonly Finding[] | undefined; + setFileResult(fileName: string, configHash: string, result: ReadonlyArray): void; save(): void; } export {}; diff --git a/baselines/packages/ymir/api/src/index.d.ts b/baselines/packages/ymir/api/src/index.d.ts index d300f7fbb..b50857bdb 100644 --- a/baselines/packages/ymir/api/src/index.d.ts +++ b/baselines/packages/ymir/api/src/index.d.ts @@ -397,7 +397,7 @@ export declare namespace StaticProgramState { * May contain the current file. * This list is ordered by the hash of the files ascending, */ - readonly dependencies: Readonly>; + readonly dependencies?: Readonly>; /** The list of findings if this file has up-to-date results */ readonly result?: readonly Finding[]; /** Hash of the configuration used to produce `result` for this file */ diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index b121183ad..ae004cd4f 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -453,7 +453,7 @@ function createConfigHash(config: ReducedConfiguration, linterOptions: LinterOpt return '' + djb2(JSON.stringify({ rules: mapToObject(config.rules, stripRuleConfig), settings: mapToObject(config.settings, identity), - reportUselessDirectives: linterOptions.reportUselessDirectives, + ...linterOptions, })); } diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 34a8fb829..cbcf3326e 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -212,8 +212,13 @@ class ProgramStateImpl implements ProgramState { let childCount = 0; const old = oldState.files[index]; const dependencies = this.resolver.getDependencies(fileName); - const keys = Object.keys(old.dependencies); + if (old.dependencies === undefined) { + if (dependencies.size !== 0) + return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); + break processFile; + } + const keys = Object.keys(old.dependencies); if (dependencies.size !== keys.length) return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); for (const key of keys) { @@ -293,6 +298,8 @@ class ProgramStateImpl implements ProgramState { const lookup: Record = {}; const mapToIndex = ({fileName}: {fileName: string}) => lookup[this.relativePathNames.get(fileName)!]; const mapDependencies = (dependencies: ReadonlyMap) => { + if (dependencies.size === 0) + return; const result: Record = {}; for (const [key, f] of dependencies) result[key] = f === null diff --git a/packages/ymir/src/index.ts b/packages/ymir/src/index.ts index 2229ada76..c5ab8f358 100644 --- a/packages/ymir/src/index.ts +++ b/packages/ymir/src/index.ts @@ -551,7 +551,7 @@ export namespace StaticProgramState { * May contain the current file. * This list is ordered by the hash of the files ascending, */ - readonly dependencies: Readonly>; + readonly dependencies?: Readonly>; /** The list of findings if this file has up-to-date results */ readonly result?: readonly Finding[]; /** Hash of the configuration used to produce `result` for this file */ From 97dd546a85f2bcdbc4cbb1dbb869ed29ad147fd9 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 21:11:39 +0100 Subject: [PATCH 17/37] correctly mark files without dependencies as up-to-date --- packages/wotan/src/services/program-state.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index cbcf3326e..95aa6742d 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -212,20 +212,15 @@ class ProgramStateImpl implements ProgramState { let childCount = 0; const old = oldState.files[index]; const dependencies = this.resolver.getDependencies(fileName); - if (old.dependencies === undefined) { - if (dependencies.size !== 0) - return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - break processFile; - } + const keys = old.dependencies === undefined ? [] : Object.keys(old.dependencies); - const keys = Object.keys(old.dependencies); if (dependencies.size !== keys.length) return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); for (const key of keys) { let newDeps = dependencies.get(key); if (newDeps === undefined) return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); - const oldDeps = old.dependencies[key]; + const oldDeps = old.dependencies![key]; if (oldDeps === null) { if (newDeps !== null) return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); From c2eafaf67431472573f77fc94726ca379951042b Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 21:35:05 +0100 Subject: [PATCH 18/37] use emptyArray constant where possible plus drive-by fix in linter to avoid unnecessary work if no rules are enabled --- packages/wotan/src/argparse.ts | 13 ++++++------- packages/wotan/src/linter.ts | 9 +++++---- .../wotan/src/services/default/builtin-resolver.ts | 3 ++- packages/wotan/src/services/default/file-filter.ts | 4 ++-- packages/wotan/src/services/program-state.ts | 4 ++-- packages/wotan/src/utils.ts | 4 +++- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/wotan/src/argparse.ts b/packages/wotan/src/argparse.ts index b30214e42..040b64ce6 100644 --- a/packages/wotan/src/argparse.ts +++ b/packages/wotan/src/argparse.ts @@ -3,6 +3,7 @@ import { ConfigurationError, Format, GlobalOptions, Severity } from '@fimbul/ymi import { LintOptions } from './runner'; import debug = require('debug'); import { OptionParser } from './optparse'; +import { emptyArray } from './utils'; const log = debug('wotan:argparse'); @@ -47,11 +48,11 @@ export interface ParsedGlobalOptions extends LintOptions { } export const GLOBAL_OPTIONS_SPEC = { - modules: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), []), + modules: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), emptyArray), config: OptionParser.Factory.parsePrimitive('string'), - files: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), []), - exclude: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), []), - project: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), []), + files: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), emptyArray), + exclude: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), emptyArray), + project: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitiveOrArray('string'), emptyArray), references: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitive('boolean'), false), cache: OptionParser.Transform.withDefault(OptionParser.Factory.parsePrimitive('boolean'), false), formatter: OptionParser.Factory.parsePrimitive('string'), @@ -249,9 +250,7 @@ function parseShowCommand(args: string[], defaults: ParsedGlobalOptions): ShowCo break; case '-m': case '--module': - if (modules === undefined) - modules = []; - modules.push(...expectStringArgument(args, ++i, arg).split(/,/g).filter(isTruthy)); + (modules ??= []).push(...expectStringArgument(args, ++i, arg).split(/,/g).filter(isTruthy)); break; case '--': files.push(...args.slice(i + 1).filter(isTruthy)); diff --git a/packages/wotan/src/linter.ts b/packages/wotan/src/linter.ts index fc3727dbb..0b7417a2c 100644 --- a/packages/wotan/src/linter.ts +++ b/packages/wotan/src/linter.ts @@ -18,7 +18,7 @@ import { applyFixes } from './fix'; import * as debug from 'debug'; import { injectable } from 'inversify'; import { RuleLoader } from './services/rule-loader'; -import { calculateChangeRange, invertChangeRange, mapDefined } from './utils'; +import { calculateChangeRange, emptyArray, invertChangeRange, mapDefined } from './utils'; import { ConvertedAst, convertAst, isCompilerOptionEnabled, getTsCheckDirective } from 'tsutils'; const log = debug('wotan:linter'); @@ -182,14 +182,15 @@ export class Linter { log('No active rules'); if (options.reportUselessDirectives !== undefined) { findings = this.filterFactory - .create({sourceFile, getWrappedAst() { return convertAst(sourceFile).wrapped; }, ruleNames: []}) + .create({sourceFile, getWrappedAst() { return convertAst(sourceFile).wrapped; }, ruleNames: emptyArray}) .reportUseless(options.reportUselessDirectives); log('Found %d useless directives', findings.length); } else { - findings = []; + findings = emptyArray; } + } else { + findings = this.applyRules(sourceFile, programFactory, rules, config.settings, options); } - findings = this.applyRules(sourceFile, programFactory, rules, config.settings, options); return processor === undefined ? findings : processor.postprocess(findings); } diff --git a/packages/wotan/src/services/default/builtin-resolver.ts b/packages/wotan/src/services/default/builtin-resolver.ts index cefa35497..6504b545f 100644 --- a/packages/wotan/src/services/default/builtin-resolver.ts +++ b/packages/wotan/src/services/default/builtin-resolver.ts @@ -1,12 +1,13 @@ import { injectable } from 'inversify'; import { BuiltinResolver, Resolver } from '@fimbul/ymir'; import * as path from 'path'; +import { emptyArray } from '../../utils'; @injectable() export class DefaultBuiltinResolver implements BuiltinResolver { private get builtinPackagePath() { const resolved = path.dirname( - this.resolver.resolve('@fimbul/mimir', path.join(__dirname, '../'.repeat(/*offset to package root*/ 3)), []), + this.resolver.resolve('@fimbul/mimir', path.join(__dirname, '../'.repeat(/*offset to package root*/ 3)), emptyArray), ); Object.defineProperty(this, 'builtinPackagePath', {value: resolved}); diff --git a/packages/wotan/src/services/default/file-filter.ts b/packages/wotan/src/services/default/file-filter.ts index 995e32a1c..9b2b86b40 100644 --- a/packages/wotan/src/services/default/file-filter.ts +++ b/packages/wotan/src/services/default/file-filter.ts @@ -1,7 +1,7 @@ import { injectable } from 'inversify'; import { FileFilterFactory, FileFilterContext, FileFilter } from '@fimbul/ymir'; import * as ts from 'typescript'; -import { unixifyPath, getOutputFileNamesOfProjectReference, iterateProjectReferences, flatMap } from '../../utils'; +import { unixifyPath, getOutputFileNamesOfProjectReference, iterateProjectReferences, flatMap, emptyArray } from '../../utils'; import * as path from 'path'; @injectable() @@ -45,7 +45,7 @@ class DefaultFileFilter implements FileFilter { this.typeRoots = ts.getEffectiveTypeRoots(this.options, { directoryExists: (dir) => this.host.directoryExists(dir), getCurrentDirectory: () => this.program.getCurrentDirectory(), - }) || []; + }) || emptyArray; return !this.typeRoots.every((typeRoot) => path.relative(typeRoot, fileName).startsWith('..' + path.sep)); } diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 95aa6742d..697b0be06 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -1,7 +1,7 @@ import { injectable } from 'inversify'; import * as ts from 'typescript'; import { DependencyResolver, DependencyResolverFactory, DependencyResolverHost } from './dependency-resolver'; -import { resolveCachedResult, djb2, unixifyPath } from '../utils'; +import { resolveCachedResult, djb2, unixifyPath, emptyArray } from '../utils'; import bind from 'bind-decorator'; import { Finding, StatePersistence, StaticProgramState } from '@fimbul/ymir'; import debug = require('debug'); @@ -212,7 +212,7 @@ class ProgramStateImpl implements ProgramState { let childCount = 0; const old = oldState.files[index]; const dependencies = this.resolver.getDependencies(fileName); - const keys = old.dependencies === undefined ? [] : Object.keys(old.dependencies); + const keys = old.dependencies === undefined ? emptyArray : Object.keys(old.dependencies); if (dependencies.size !== keys.length) return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); diff --git a/packages/wotan/src/utils.ts b/packages/wotan/src/utils.ts index 564b6d870..0abe04c7d 100644 --- a/packages/wotan/src/utils.ts +++ b/packages/wotan/src/utils.ts @@ -13,11 +13,13 @@ import * as path from 'path'; */ export const OFFSET_TO_NODE_MODULES = 3; +export const emptyArray: readonly never[] = []; + export function arrayify(maybeArr: T | ReadonlyArray | undefined): ReadonlyArray { return Array.isArray(maybeArr) ? maybeArr : maybeArr === undefined - ? [] + ? emptyArray : [maybeArr]; } From f460075830449ba751a4aac416379fa33cc689b5 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sat, 6 Feb 2021 21:43:10 +0100 Subject: [PATCH 19/37] fix build --- packages/wotan/src/runner.ts | 2 -- packages/wotan/test/argparse.spec.ts | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index ae004cd4f..a88c8b6a2 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -448,7 +448,6 @@ function shouldFix(sourceFile: ts.SourceFile, options: Pick, return options.fix; } - function createConfigHash(config: ReducedConfiguration, linterOptions: LinterOptions) { return '' + djb2(JSON.stringify({ rules: mapToObject(config.rules, stripRuleConfig), @@ -472,7 +471,6 @@ function stripRuleConfig({rulesDirectories: _ignored, ...rest}: EffectiveConfigu return rest; } - declare module 'typescript' { function matchFiles( path: string, diff --git a/packages/wotan/test/argparse.spec.ts b/packages/wotan/test/argparse.spec.ts index 9bda27a51..4ca70cb9a 100644 --- a/packages/wotan/test/argparse.spec.ts +++ b/packages/wotan/test/argparse.spec.ts @@ -786,12 +786,12 @@ test('parses lint command', (t) => { files: [], exclude: [], formatter: undefined, - project: ['.'], + project: [], references: false, fix: false, extensions: undefined, reportUselessDirectives: false, - cache: false, + cache: true, }, 'parses --cache with implicit --project', ); From 0e23eba91717bec696d89fa6e3a9262fbc7782e5 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sun, 7 Feb 2021 19:07:52 +0100 Subject: [PATCH 20/37] export new services from public api --- packages/wotan/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/wotan/index.ts b/packages/wotan/index.ts index b58595780..989fb45ce 100644 --- a/packages/wotan/index.ts +++ b/packages/wotan/index.ts @@ -11,10 +11,13 @@ export * from './src/services/default/line-switches'; export * from './src/services/default/message-handler'; export * from './src/services/default/resolver'; export * from './src/services/default/rule-loader-host'; +export * from './src/services/default/state-persistence'; export * from './src/services/cached-file-system'; export * from './src/services/configuration-manager'; +export * from './src/services/dependency-resolver'; export * from './src/services/formatter-loader'; export * from './src/services/processor-loader'; +export * from './src/services/program-state'; export * from './src/services/rule-loader'; export { parseGlobalOptions, ParsedGlobalOptions, GLOBAL_OPTIONS_SPEC } from './src/argparse'; export * from './src/baseline'; From 8444a020c1b9e5354a49a5e8d8f21496bb8cde70 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sun, 7 Feb 2021 19:19:49 +0100 Subject: [PATCH 21/37] test DependencyResolver --- baselines/packages/wotan/api/index.d.ts | 3 + packages/wotan/package.json | 1 + packages/wotan/src/project-host.ts | 25 +- .../wotan/src/services/dependency-resolver.ts | 8 +- .../wotan/test/dependency-resolver.spec.ts | 246 ++++++++++++++++++ yarn.lock | 12 + 6 files changed, 282 insertions(+), 13 deletions(-) create mode 100644 packages/wotan/test/dependency-resolver.spec.ts diff --git a/baselines/packages/wotan/api/index.d.ts b/baselines/packages/wotan/api/index.d.ts index b58595780..989fb45ce 100644 --- a/baselines/packages/wotan/api/index.d.ts +++ b/baselines/packages/wotan/api/index.d.ts @@ -11,10 +11,13 @@ export * from './src/services/default/line-switches'; export * from './src/services/default/message-handler'; export * from './src/services/default/resolver'; export * from './src/services/default/rule-loader-host'; +export * from './src/services/default/state-persistence'; export * from './src/services/cached-file-system'; export * from './src/services/configuration-manager'; +export * from './src/services/dependency-resolver'; export * from './src/services/formatter-loader'; export * from './src/services/processor-loader'; +export * from './src/services/program-state'; export * from './src/services/rule-loader'; export { parseGlobalOptions, ParsedGlobalOptions, GLOBAL_OPTIONS_SPEC } from './src/argparse'; export * from './src/baseline'; diff --git a/packages/wotan/package.json b/packages/wotan/package.json index dcd728741..a6c300709 100644 --- a/packages/wotan/package.json +++ b/packages/wotan/package.json @@ -37,6 +37,7 @@ "@types/resolve": "^1.14.0", "@types/semver": "^7.0.0", "escape-string-regexp": "^4.0.0", + "memfs": "^3.2.0", "rimraf": "^3.0.0" }, "dependencies": { diff --git a/packages/wotan/src/project-host.ts b/packages/wotan/src/project-host.ts index 909e09bfb..e7e119209 100644 --- a/packages/wotan/src/project-host.ts +++ b/packages/wotan/src/project-host.ts @@ -95,33 +95,34 @@ export class ProjectHost implements ts.CompilerHost { return result; } for (const entry of entries) { - const fileName = `${dir}/${entry.name}`; switch (entry.kind) { case FileKind.File: { + const fileName = `${dir}/${entry.name}`; if (!hasSupportedExtension(fileName, additionalExtensions)) { const c = this.config || this.tryFindConfig(fileName); const processor = c && this.configManager.getProcessor(c, fileName); if (processor) { const ctor = this.processorLoader.loadProcessor(processor); - const newName = fileName + - ctor.getSuffixForFile({ - fileName, - getSettings: () => this.configManager.getSettings(c!, fileName), - readFile: () => this.fs.readFile(fileName), - }); + const suffix = ctor.getSuffixForFile({ + fileName, + getSettings: () => this.configManager.getSettings(c!, fileName), + readFile: () => this.fs.readFile(fileName), + }); + const newName = fileName + suffix; + if (hasSupportedExtension(newName, additionalExtensions)) { - files.push(newName); + files.push(entry.name + suffix); this.reverseMap.set(newName, fileName); break; } } } - files.push(fileName); + files.push(entry.name); this.files.push(fileName); break; } case FileKind.Directory: - directories.push(fileName); + directories.push(entry.name); } } return result; @@ -212,10 +213,10 @@ export class ProjectHost implements ts.CompilerHost { public getDirectories(dir: string) { const cached = this.directoryEntries.get(dir); if (cached !== undefined) - return cached.directories.map((d) => path.posix.basename(d)); + return cached.directories.slice(); return mapDefined( this.fs.readDirectory(dir), - (entry) => entry.kind === FileKind.Directory ? path.join(dir, entry.name) : undefined, + (entry) => entry.kind === FileKind.Directory ? entry.name : undefined, ); } public getSourceFile(fileName: string, languageVersion: ts.ScriptTarget.JSON): ts.JsonSourceFile | undefined; diff --git a/packages/wotan/src/services/dependency-resolver.ts b/packages/wotan/src/services/dependency-resolver.ts index 05df7113f..eb69024df 100644 --- a/packages/wotan/src/services/dependency-resolver.ts +++ b/packages/wotan/src/services/dependency-resolver.ts @@ -104,6 +104,11 @@ class DependencyResolverImpl implements DependencyResolver { public getDependencies(file: string) { this.state ??= this.buildState(); const result = new Map(); + { + const augmentations = this.state.moduleAugmentations.get(file); + if (augmentations !== undefined) + result.set('\0', augmentations); + } for (const [identifier, resolved] of this.getExternalReferences(file)) { const filesAffectingAmbientModule = this.state.ambientModules.get(identifier); if (filesAffectingAmbientModule !== undefined) { @@ -250,7 +255,7 @@ function getSourceOfProjectReferenceRedirect(outputFileName: string, ref: ts.Res const projectDirectory = path.dirname(ref.sourceFile.fileName); const origin = unixifyPath(path.resolve( options.rootDir || projectDirectory, - path.relative(options.declarationDir || options.outDir || projectDirectory, outputFileName.slice(0, -5)), + path.relative(options.declarationDir || options.outDir || /* istanbul ignore next */ projectDirectory, outputFileName.slice(0, -5)), )); for (const extension of ['.ts', '.tsx', '.js', '.jsx']) { @@ -258,5 +263,6 @@ function getSourceOfProjectReferenceRedirect(outputFileName: string, ref: ts.Res if (ref.commandLine.fileNames.includes(name)) return name; } + /* istanbul ignore next */ return outputFileName; // should never happen } diff --git a/packages/wotan/test/dependency-resolver.spec.ts b/packages/wotan/test/dependency-resolver.spec.ts new file mode 100644 index 000000000..f96faf1ca --- /dev/null +++ b/packages/wotan/test/dependency-resolver.spec.ts @@ -0,0 +1,246 @@ +import 'reflect-metadata'; +import test from 'ava'; +import { Dirent } from 'fs'; +import { DirectoryJSON, Volume } from 'memfs'; +import * as ts from 'typescript'; +import { DependencyResolverFactory, DependencyResolverHost } from '../src/services/dependency-resolver'; +import { mapDefined, unixifyPath } from '../src/utils'; +import * as path from 'path'; + +function identity(v: T) { + return v; +} + +function setup(fileContents: DirectoryJSON, useSourceOfProjectReferenceRedirect?: boolean) { + const {root} = path.parse(unixifyPath(process.cwd())); + const vol = Volume.fromJSON(fileContents, root); + function fileExists(f: string) { + try { + return vol.statSync(f).isFile(); + } catch { + return false; + } + } + function readFile(f: string) { + try { + return vol.readFileSync(f, {encoding: 'utf8'}).toString(); + } catch { + return; + } + } + function readDirectory( + rootDir: string, + extensions: readonly string[], + excludes: readonly string[] | undefined, + includes: readonly string[], + depth?: number, + ) { + return ts.matchFiles( + rootDir, + extensions, + excludes, + includes, + true, + root, + depth, + (dir) => { + const files: string[] = []; + const directories: string[] = []; + const result: ts.FileSystemEntries = {files, directories}; + let entries; + try { + entries = vol.readdirSync(dir, {withFileTypes: true, encoding: 'utf8'}); + } catch { + return result; + } + for (const entry of entries) + (entry.isFile() ? files : directories).push(entry.name); + return result; + }, + identity, + ); + } + + const commandLine = ts.getParsedCommandLineOfConfigFile(root + 'tsconfig.json', {}, { + fileExists, + readFile, + readDirectory, + getCurrentDirectory() { return root; }, + useCaseSensitiveFileNames: true, + onUnRecoverableConfigFileDiagnostic() {}, + })!; + const resolutionCache = ts.createModuleResolutionCache(root, identity, commandLine.options); + const compilerHost: ts.CompilerHost & DependencyResolverHost = { + fileExists, + readFile, + readDirectory, + directoryExists(d) { + try { + return vol.statSync(d).isDirectory(); + } catch { + return false; + } + }, + getCanonicalFileName: identity, + getCurrentDirectory() { + return root; + }, + getNewLine() { + return '\n'; + }, + useCaseSensitiveFileNames() { + return true; + }, + useSourceOfProjectReferenceRedirect: useSourceOfProjectReferenceRedirect === undefined + ? undefined + : () => useSourceOfProjectReferenceRedirect, + getDefaultLibFileName: ts.getDefaultLibFilePath, + getSourceFile(fileName, languageVersion) { + const content = readFile(fileName); + return content === undefined ? undefined : ts.createSourceFile(fileName, content, languageVersion, true); + }, + writeFile() { + throw new Error('not implemented'); + }, + realpath: identity, + getDirectories(d) { + return mapDefined(vol.readdirSync(d, {encoding: 'utf8'}), (f) => { + return this.directoryExists!(d + '/' + f) ? f : undefined; + }); + }, + resolveModuleNames(modules, containingFile, _, redirectedReference, o) { + return modules.map( + (m) => ts.resolveModuleName(m, containingFile, o, this, resolutionCache, redirectedReference).resolvedModule); + }, + }; + const program = ts.createProgram({ + host: compilerHost, + options: commandLine.options, + rootNames: commandLine.fileNames, + projectReferences: commandLine.projectReferences, + }); + const dependencyResolver = new DependencyResolverFactory().create(compilerHost, program); + return {vol, compilerHost, program, dependencyResolver, root}; +} + +test('resolves imports', (t) => { + let {dependencyResolver, program, compilerHost, vol, root} = setup({ + 'tsconfig.json': JSON.stringify({compilerOptions: {moduleResolution: 'node', allowJs: true}}), + 'a.ts': 'import * as a from "./a"; import {b} from "./b"; import {c} from "c"; import {d} from "d"; import {e} from "e"; import {f} from "f"; import {f1} from "f1"; import {f2} from "f2"', + 'b.ts': 'export const b = 1;', + 'node_modules/c/index.d.ts': 'export const c = 1;', + 'node_modules/f1/index.d.ts': 'export const f1 = 1;', + 'empty.js': '', + 'ambient.ts': 'declare module "d" {export const d: number;}', + 'global.ts': 'declare var v = 1;', + 'other.ts': 'export {}; declare module "c" { export let other: number; }; declare module "d" {}; declare module "f1" {}; declare module "f2" {}; declare module "goo" {};', + 'pattern.ts': 'declare module "f*"; declare module "goo*oo";', + 'declare-global.ts': 'export {}; declare global {}', + 'umd.d.ts': 'export let stuff: number; export as namespace Stuff;', + }); + + t.deepEqual(dependencyResolver.getFilesAffectingGlobalScope(), [root + 'declare-global.ts', root + 'global.ts', root + 'umd.d.ts']); + t.deepEqual(dependencyResolver.getDependencies(root + 'a.ts'), new Map([ + ['./a', [root + 'a.ts']], + ['./b', [root + 'b.ts']], + ['c', [root + 'node_modules/c/index.d.ts', root + 'other.ts']], + ['d', [root + 'ambient.ts', root + 'other.ts']], + ['e', null], + ['f', [root + 'pattern.ts', root + 'other.ts']], + ['f1', [root + 'node_modules/f1/index.d.ts', root + 'other.ts']], + ['f2', [root + 'pattern.ts', root + 'other.ts']], + ])); + t.deepEqual(dependencyResolver.getDependencies(root + 'b.ts'), new Map()); + t.deepEqual(dependencyResolver.getDependencies(root + 'node_modules/c/index.d.ts'), new Map([['\0', [root + 'other.ts']]])); + t.deepEqual(dependencyResolver.getDependencies(root + 'node_modules/f1/index.d.ts'), new Map([['\0', [root + 'other.ts']]])); + t.deepEqual(dependencyResolver.getDependencies(root + 'empty.js'), new Map()); + t.deepEqual(dependencyResolver.getDependencies(root + 'ambient.ts'), new Map([['d', [root + 'ambient.ts', root + 'other.ts']]])); + t.deepEqual(dependencyResolver.getDependencies(root + 'global.ts'), new Map()); + t.deepEqual(dependencyResolver.getDependencies(root + 'declare-global.ts'), new Map()); + t.deepEqual(dependencyResolver.getDependencies(root + 'other.ts'), new Map([ + ['c', [root + 'node_modules/c/index.d.ts', root + 'other.ts']], + ['d', [root + 'ambient.ts', root + 'other.ts']], + ['f1', [root + 'node_modules/f1/index.d.ts', root + 'other.ts']], + ['f2', [root + 'pattern.ts', root + 'other.ts']], + ['goo', null], + ])); + t.deepEqual(dependencyResolver.getDependencies(root + 'pattern.ts'), new Map([ + ['f*', [root + 'pattern.ts', root + 'other.ts']], + ['goo*oo', [root + 'pattern.ts']], + ])); + t.deepEqual(dependencyResolver.getDependencies(root + 'umd.d.ts'), new Map()); + + vol.writeFileSync(root + 'empty.js', '/** @type {import("./b").b} */ var f = require("e");', {encoding: 'utf8'}); + program = ts.createProgram({ + oldProgram: program, + host: compilerHost, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + }); + dependencyResolver.update(program, root + 'empty.js'); + + t.deepEqual(dependencyResolver.getDependencies(root + 'empty.js'), new Map([ + ['./b', [root + 'b.ts']], + ['e', null], + ])); + t.deepEqual( + dependencyResolver.getFilesAffectingGlobalScope(), + [root + 'declare-global.ts', root + 'empty.js', root + 'global.ts', root + 'umd.d.ts'], + ); +}); + +test('handles useSourceOfProjectReferenceRedirect', (t) => { + const {dependencyResolver, root} = setup( + { + 'tsconfig.json': JSON.stringify({references: [{path: 'a'}, {path: 'b'}, {path: 'c'}], include: ['src']}), + 'src/a.ts': 'export * from "../a/decl/src/a"; export * from "../a/decl/src/b"; export * from "../a/decl/src/c"; export * from "../a/decl/src/d";', + 'src/b.ts': 'export * from "../b/dist/file";', + 'src/c.ts': 'import "../c/dist/outfile', + 'src/d.ts': 'import "../d/file', + 'a/tsconfig.json': JSON.stringify( + {compilerOptions: {composite: true, allowJs: true, outDir: 'dist', declarationDir: 'decl'}, include: ['src']}, + ), + 'a/src/a.ts': 'export {}', + 'a/src/b.tsx': 'export {}', + 'a/src/c.js': 'export {}', + 'a/src/d.jsx': 'export {}', + 'b/tsconfig.json': JSON.stringify({compilerOptions: {composite: true, outDir: 'dist', rootDir: 'src'}}), + 'b/src/file.ts': 'export {};', + 'c/tsconfig.json': JSON.stringify({compilerOptions: {composite: true, outFile: 'dist/outfile.js'}, files: ['a.ts', 'b.ts']}), + 'c/a.ts': 'namespace foo {}', + 'c/b.ts': 'namespace foo {}', + 'd/tsconfig.json': JSON.stringify({compilerOptions: {composite: true}, files: ['file.ts']}), + 'd/file.ts': 'export {}', + }, + true, + ); + + t.deepEqual(dependencyResolver.getDependencies(root + 'src/a.ts'), new Map([ + ['../a/decl/src/a', [root + 'a/src/a.ts']], + ['../a/decl/src/b', [root + 'a/src/b.tsx']], + ['../a/decl/src/c', [root + 'a/src/c.js']], + ['../a/decl/src/d', [root + 'a/src/d.jsx']], + ])); + t.deepEqual(dependencyResolver.getDependencies(root + 'src/b.ts'), new Map([['../b/dist/file', [root + 'b/src/file.ts']]])); + t.deepEqual(dependencyResolver.getDependencies(root + 'src/c.ts'), new Map([['../c/dist/outfile', [root + 'c/a.ts']]])); + t.deepEqual(dependencyResolver.getDependencies(root + 'src/d.ts'), new Map([['../d/file', [root + 'd/file.ts']]])); +}); + +test('handles disableSourceOfProjectReferenceRedirect', (t) => { + const {dependencyResolver, root} = setup( + { + 'tsconfig.json': JSON.stringify( + {references: [{path: 'a'}], compilerOptions: {disableSourceOfProjectReferenceRedirect: true}, include: ['src']}, + ), + 'src/a.ts': 'export * from "../a/dist/file";', + 'a/tsconfig.json': JSON.stringify({compilerOptions: {composite: true, outDir: 'dist', rootDir: 'src'}}), + 'a/src/file.ts': 'export {};', + 'a/dist/file.d.ts': 'export {};', + }, + true, + ); + + t.deepEqual(dependencyResolver.getDependencies(root + 'src/a.ts'), new Map([ + ['../a/dist/file', [root + 'a/dist/file.d.ts']], + ])); +}); diff --git a/yarn.lock b/yarn.lock index 194637599..704fd15e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1642,6 +1642,11 @@ fromentries@^1.2.0: resolved "https://registry.yarnpkg.com/fromentries/-/fromentries-1.3.2.tgz#e4bca6808816bf8f93b52750f1127f5a6fd86e3a" integrity sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg== +fs-monkey@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.1.tgz#4a82f36944365e619f4454d9fff106553067b781" + integrity sha512-fcSa+wyTqZa46iWweI7/ZiUfegOZl0SG8+dltIwFXo7+zYU9J9kpS3NB6pZcSlJdhvIwp81Adx2XhZorncxiaA== + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -2513,6 +2518,13 @@ mem@^8.0.0: map-age-cleaner "^0.1.3" mimic-fn "^3.1.0" +memfs@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.2.0.tgz#f9438e622b5acd1daa8a4ae160c496fdd1325b26" + integrity sha512-f/xxz2TpdKv6uDn6GtHee8ivFyxwxmPuXatBb1FBwxYNuVpbM3k/Y1Z+vC0mH/dIXXrukYfe3qe5J32Dfjg93A== + dependencies: + fs-monkey "1.0.1" + memory-fs@^0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.5.0.tgz#324c01288b88652966d161db77838720845a8e3c" From b368c5453938a771153d67f00e7ef86e4204cf36 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Sun, 7 Feb 2021 20:46:15 +0100 Subject: [PATCH 22/37] more testing for dependency resolver --- .../wotan/test/dependency-resolver.spec.ts | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/wotan/test/dependency-resolver.spec.ts b/packages/wotan/test/dependency-resolver.spec.ts index f96faf1ca..259762aa2 100644 --- a/packages/wotan/test/dependency-resolver.spec.ts +++ b/packages/wotan/test/dependency-resolver.spec.ts @@ -125,7 +125,7 @@ function setup(fileContents: DirectoryJSON, useSourceOfProjectReferenceRedirect? test('resolves imports', (t) => { let {dependencyResolver, program, compilerHost, vol, root} = setup({ - 'tsconfig.json': JSON.stringify({compilerOptions: {moduleResolution: 'node', allowJs: true}}), + 'tsconfig.json': JSON.stringify({compilerOptions: {moduleResolution: 'node', allowJs: true}, exclude: ['excluded.ts']}), 'a.ts': 'import * as a from "./a"; import {b} from "./b"; import {c} from "c"; import {d} from "d"; import {e} from "e"; import {f} from "f"; import {f1} from "f1"; import {f2} from "f2"', 'b.ts': 'export const b = 1;', 'node_modules/c/index.d.ts': 'export const c = 1;', @@ -137,6 +137,7 @@ test('resolves imports', (t) => { 'pattern.ts': 'declare module "f*"; declare module "goo*oo";', 'declare-global.ts': 'export {}; declare global {}', 'umd.d.ts': 'export let stuff: number; export as namespace Stuff;', + 'excluded.ts': 'declare module "e" {}; declare module "d" {};', }); t.deepEqual(dependencyResolver.getFilesAffectingGlobalScope(), [root + 'declare-global.ts', root + 'global.ts', root + 'umd.d.ts']); @@ -187,6 +188,28 @@ test('resolves imports', (t) => { dependencyResolver.getFilesAffectingGlobalScope(), [root + 'declare-global.ts', root + 'empty.js', root + 'global.ts', root + 'umd.d.ts'], ); + + vol.appendFileSync(root + 'b.ts', 'import "./excluded";'); + program = ts.createProgram({ + oldProgram: program, + host: compilerHost, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + }); + dependencyResolver.update(program, root + 'b.ts'); + + t.deepEqual(dependencyResolver.getDependencies(root + 'b.ts'), new Map([['./excluded', [root + 'excluded.ts']]])); + t.deepEqual(dependencyResolver.getDependencies(root + 'a.ts'), new Map([ + ['./a', [root + 'a.ts']], + ['./b', [root + 'b.ts']], + ['c', [root + 'node_modules/c/index.d.ts', root + 'other.ts']], + ['d', [root + 'excluded.ts', root + 'ambient.ts', root + 'other.ts']], + ['e', [root + 'excluded.ts']], + ['f', [root + 'pattern.ts', root + 'other.ts']], + ['f1', [root + 'node_modules/f1/index.d.ts', root + 'other.ts']], + ['f2', [root + 'pattern.ts', root + 'other.ts']], + ])); + t.deepEqual(dependencyResolver.getDependencies(root + 'ambient.ts'), new Map([['d', [root + 'excluded.ts', root + 'ambient.ts', root + 'other.ts']]])); }); test('handles useSourceOfProjectReferenceRedirect', (t) => { From c8445f0241fedaafebea65c0b15b26f0b89d6617 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Mon, 8 Feb 2021 20:26:32 +0100 Subject: [PATCH 23/37] add test for ambient module overruling resolved module --- .../wotan/test/dependency-resolver.spec.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/wotan/test/dependency-resolver.spec.ts b/packages/wotan/test/dependency-resolver.spec.ts index 259762aa2..619918f45 100644 --- a/packages/wotan/test/dependency-resolver.spec.ts +++ b/packages/wotan/test/dependency-resolver.spec.ts @@ -126,14 +126,15 @@ function setup(fileContents: DirectoryJSON, useSourceOfProjectReferenceRedirect? test('resolves imports', (t) => { let {dependencyResolver, program, compilerHost, vol, root} = setup({ 'tsconfig.json': JSON.stringify({compilerOptions: {moduleResolution: 'node', allowJs: true}, exclude: ['excluded.ts']}), - 'a.ts': 'import * as a from "./a"; import {b} from "./b"; import {c} from "c"; import {d} from "d"; import {e} from "e"; import {f} from "f"; import {f1} from "f1"; import {f2} from "f2"', + 'a.ts': 'import * as a from "./a"; import {b} from "./b"; import {c} from "c"; import {d} from "d"; import {e} from "e"; import {f} from "f"; import {f1} from "f1"; import {f2} from "f2"; import {g} from "g";', 'b.ts': 'export const b = 1;', 'node_modules/c/index.d.ts': 'export const c = 1;', 'node_modules/f1/index.d.ts': 'export const f1 = 1;', + 'node_modules/g/index.d.ts': 'export const g = 1;', 'empty.js': '', - 'ambient.ts': 'declare module "d" {export const d: number;}', + 'ambient.ts': 'declare module "d" {export const d: number;} declare module "g" {}', 'global.ts': 'declare var v = 1;', - 'other.ts': 'export {}; declare module "c" { export let other: number; }; declare module "d" {}; declare module "f1" {}; declare module "f2" {}; declare module "goo" {};', + 'other.ts': 'export {}; declare module "c" { export let other: number; }; declare module "d" {}; declare module "f1" {}; declare module "f2" {}; declare module "goo" {}; declare module "g" {};', 'pattern.ts': 'declare module "f*"; declare module "goo*oo";', 'declare-global.ts': 'export {}; declare global {}', 'umd.d.ts': 'export let stuff: number; export as namespace Stuff;', @@ -150,12 +151,16 @@ test('resolves imports', (t) => { ['f', [root + 'pattern.ts', root + 'other.ts']], ['f1', [root + 'node_modules/f1/index.d.ts', root + 'other.ts']], ['f2', [root + 'pattern.ts', root + 'other.ts']], + ['g', [root + 'ambient.ts', root + 'other.ts']], ])); t.deepEqual(dependencyResolver.getDependencies(root + 'b.ts'), new Map()); t.deepEqual(dependencyResolver.getDependencies(root + 'node_modules/c/index.d.ts'), new Map([['\0', [root + 'other.ts']]])); t.deepEqual(dependencyResolver.getDependencies(root + 'node_modules/f1/index.d.ts'), new Map([['\0', [root + 'other.ts']]])); t.deepEqual(dependencyResolver.getDependencies(root + 'empty.js'), new Map()); - t.deepEqual(dependencyResolver.getDependencies(root + 'ambient.ts'), new Map([['d', [root + 'ambient.ts', root + 'other.ts']]])); + t.deepEqual(dependencyResolver.getDependencies(root + 'ambient.ts'), new Map([ + ['d', [root + 'ambient.ts', root + 'other.ts']], + ['g', [root + 'ambient.ts', root + 'other.ts']], + ])); t.deepEqual(dependencyResolver.getDependencies(root + 'global.ts'), new Map()); t.deepEqual(dependencyResolver.getDependencies(root + 'declare-global.ts'), new Map()); t.deepEqual(dependencyResolver.getDependencies(root + 'other.ts'), new Map([ @@ -164,6 +169,7 @@ test('resolves imports', (t) => { ['f1', [root + 'node_modules/f1/index.d.ts', root + 'other.ts']], ['f2', [root + 'pattern.ts', root + 'other.ts']], ['goo', null], + ['g', [root + 'ambient.ts', root + 'other.ts']], ])); t.deepEqual(dependencyResolver.getDependencies(root + 'pattern.ts'), new Map([ ['f*', [root + 'pattern.ts', root + 'other.ts']], @@ -208,8 +214,12 @@ test('resolves imports', (t) => { ['f', [root + 'pattern.ts', root + 'other.ts']], ['f1', [root + 'node_modules/f1/index.d.ts', root + 'other.ts']], ['f2', [root + 'pattern.ts', root + 'other.ts']], + ['g', [root + 'ambient.ts', root + 'other.ts']], + ])); + t.deepEqual(dependencyResolver.getDependencies(root + 'ambient.ts'), new Map([ + ['d', [root + 'excluded.ts', root + 'ambient.ts', root + 'other.ts']], + ['g', [root + 'ambient.ts', root + 'other.ts']], ])); - t.deepEqual(dependencyResolver.getDependencies(root + 'ambient.ts'), new Map([['d', [root + 'excluded.ts', root + 'ambient.ts', root + 'other.ts']]])); }); test('handles useSourceOfProjectReferenceRedirect', (t) => { From 769afc050640a2b846fd99f0ee5ba2edd1326c40 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Mon, 8 Feb 2021 20:32:53 +0100 Subject: [PATCH 24/37] avoid leaking module augmentation for typescript --- baselines/packages/wotan/api/src/runner.d.ts | 8 ------- packages/wotan/src/project-host.ts | 22 ++++++++++++++++++++ packages/wotan/src/runner.ts | 19 ----------------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/baselines/packages/wotan/api/src/runner.d.ts b/baselines/packages/wotan/api/src/runner.d.ts index ac0b6fc1d..90cac8ffe 100644 --- a/baselines/packages/wotan/api/src/runner.d.ts +++ b/baselines/packages/wotan/api/src/runner.d.ts @@ -1,6 +1,5 @@ import { Linter } from './linter'; import { LintResult, DirectoryService, MessageHandler, FileFilterFactory, Severity } from '@fimbul/ymir'; -import * as ts from 'typescript'; import { ProcessorLoader } from './services/processor-loader'; import { CachedFileSystem } from './services/cached-file-system'; import { ConfigurationManager } from './services/configuration-manager'; @@ -20,10 +19,3 @@ export declare class Runner { constructor(fs: CachedFileSystem, configManager: ConfigurationManager, linter: Linter, processorLoader: ProcessorLoader, directories: DirectoryService, logger: MessageHandler, filterFactory: FileFilterFactory, programStateFactory: ProgramStateFactory); lintCollection(options: LintOptions): LintResult; } -declare module 'typescript' { - function matchFiles(path: string, extensions: ReadonlyArray, excludes: ReadonlyArray | undefined, includes: ReadonlyArray, useCaseSensitiveFileNames: boolean, currentDirectory: string, depth: number | undefined, getFileSystemEntries: (path: string) => ts.FileSystemEntries, realpath: (path: string) => string): string[]; - interface FileSystemEntries { - readonly files: ReadonlyArray; - readonly directories: ReadonlyArray; - } -} diff --git a/packages/wotan/src/project-host.ts b/packages/wotan/src/project-host.ts index e7e119209..2eaa05957 100644 --- a/packages/wotan/src/project-host.ts +++ b/packages/wotan/src/project-host.ts @@ -12,12 +12,14 @@ const log = debug('wotan:projectHost'); const additionalExtensions = ['.json']; +// @internal export interface ProcessedFileInfo { originalName: string; originalContent: string; // TODO this should move into processor because this property is never updated, but the processor is processor: AbstractProcessor; } +// @internal export class ProjectHost implements ts.CompilerHost { private reverseMap = new Map(); private files: string[] = []; @@ -305,3 +307,23 @@ export class ProjectHost implements ts.CompilerHost { return names.map((name) => resolveCachedResult(seen, name, resolve)); } } + +// @internal +declare module 'typescript' { + function matchFiles( + path: string, + extensions: ReadonlyArray, + excludes: ReadonlyArray | undefined, + includes: ReadonlyArray, + useCaseSensitiveFileNames: boolean, + currentDirectory: string, + depth: number | undefined, + getFileSystemEntries: (path: string) => ts.FileSystemEntries, + realpath: (path: string) => string, + ): string[]; + + interface FileSystemEntries { + readonly files: ReadonlyArray; + readonly directories: ReadonlyArray; + } +} diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index a88c8b6a2..1032aaca0 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -470,22 +470,3 @@ function identity(v: T) { function stripRuleConfig({rulesDirectories: _ignored, ...rest}: EffectiveConfiguration.RuleConfig) { return rest; } - -declare module 'typescript' { - function matchFiles( - path: string, - extensions: ReadonlyArray, - excludes: ReadonlyArray | undefined, - includes: ReadonlyArray, - useCaseSensitiveFileNames: boolean, - currentDirectory: string, - depth: number | undefined, - getFileSystemEntries: (path: string) => ts.FileSystemEntries, - realpath: (path: string) => string, - ): string[]; - - interface FileSystemEntries { - readonly files: ReadonlyArray; - readonly directories: ReadonlyArray; - } -} From 464a86853136888c9a9059e00cffe6fdbb2cdd14 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Tue, 9 Feb 2021 19:24:05 +0100 Subject: [PATCH 25/37] moar tests for dependency resolver --- .dependency-cruiser.json | 9 ++++++++ .../wotan/test/dependency-resolver.spec.ts | 23 +++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.dependency-cruiser.json b/.dependency-cruiser.json index 7b39d0ddf..4aa39be77 100644 --- a/.dependency-cruiser.json +++ b/.dependency-cruiser.json @@ -11,6 +11,15 @@ "path": "^packages/[^/]+/test/" } }, + { + "name": "not-to-spec", + "comment": "Don't allow dependencies to spec files", + "severity": "error", + "from": {}, + "to": { + "path": "\\.spec\\.[jt]s$" + } + }, { "name": "not-outside-package", "comment": "Don't allow packages to anything outside of packages", diff --git a/packages/wotan/test/dependency-resolver.spec.ts b/packages/wotan/test/dependency-resolver.spec.ts index 619918f45..aa903472e 100644 --- a/packages/wotan/test/dependency-resolver.spec.ts +++ b/packages/wotan/test/dependency-resolver.spec.ts @@ -1,9 +1,9 @@ import 'reflect-metadata'; -import test from 'ava'; +import test, { ExecutionContext } from 'ava'; import { Dirent } from 'fs'; import { DirectoryJSON, Volume } from 'memfs'; import * as ts from 'typescript'; -import { DependencyResolverFactory, DependencyResolverHost } from '../src/services/dependency-resolver'; +import { DependencyResolver, DependencyResolverFactory, DependencyResolverHost } from '../src/services/dependency-resolver'; import { mapDefined, unixifyPath } from '../src/utils'; import * as path from 'path'; @@ -123,6 +123,16 @@ function setup(fileContents: DirectoryJSON, useSourceOfProjectReferenceRedirect? return {vol, compilerHost, program, dependencyResolver, root}; } +function assertAllDependenciesInProgram(resolver: DependencyResolver, program: ts.Program, t: ExecutionContext) { + for (const file of program.getSourceFiles()) { + const dependencies = resolver.getDependencies(file.fileName); + for (const deps of dependencies.values()) + if (deps !== null) + for (const dep of deps) + t.not(program.getSourceFile(dep), undefined); + } +} + test('resolves imports', (t) => { let {dependencyResolver, program, compilerHost, vol, root} = setup({ 'tsconfig.json': JSON.stringify({compilerOptions: {moduleResolution: 'node', allowJs: true}, exclude: ['excluded.ts']}), @@ -176,6 +186,7 @@ test('resolves imports', (t) => { ['goo*oo', [root + 'pattern.ts']], ])); t.deepEqual(dependencyResolver.getDependencies(root + 'umd.d.ts'), new Map()); + assertAllDependenciesInProgram(dependencyResolver, program, t); vol.writeFileSync(root + 'empty.js', '/** @type {import("./b").b} */ var f = require("e");', {encoding: 'utf8'}); program = ts.createProgram({ @@ -194,6 +205,7 @@ test('resolves imports', (t) => { dependencyResolver.getFilesAffectingGlobalScope(), [root + 'declare-global.ts', root + 'empty.js', root + 'global.ts', root + 'umd.d.ts'], ); + assertAllDependenciesInProgram(dependencyResolver, program, t); vol.appendFileSync(root + 'b.ts', 'import "./excluded";'); program = ts.createProgram({ @@ -220,10 +232,11 @@ test('resolves imports', (t) => { ['d', [root + 'excluded.ts', root + 'ambient.ts', root + 'other.ts']], ['g', [root + 'ambient.ts', root + 'other.ts']], ])); + assertAllDependenciesInProgram(dependencyResolver, program, t); }); test('handles useSourceOfProjectReferenceRedirect', (t) => { - const {dependencyResolver, root} = setup( + const {dependencyResolver, program, root} = setup( { 'tsconfig.json': JSON.stringify({references: [{path: 'a'}, {path: 'b'}, {path: 'c'}], include: ['src']}), 'src/a.ts': 'export * from "../a/decl/src/a"; export * from "../a/decl/src/b"; export * from "../a/decl/src/c"; export * from "../a/decl/src/d";', @@ -257,10 +270,11 @@ test('handles useSourceOfProjectReferenceRedirect', (t) => { t.deepEqual(dependencyResolver.getDependencies(root + 'src/b.ts'), new Map([['../b/dist/file', [root + 'b/src/file.ts']]])); t.deepEqual(dependencyResolver.getDependencies(root + 'src/c.ts'), new Map([['../c/dist/outfile', [root + 'c/a.ts']]])); t.deepEqual(dependencyResolver.getDependencies(root + 'src/d.ts'), new Map([['../d/file', [root + 'd/file.ts']]])); + assertAllDependenciesInProgram(dependencyResolver, program, t); }); test('handles disableSourceOfProjectReferenceRedirect', (t) => { - const {dependencyResolver, root} = setup( + const {dependencyResolver, program, root} = setup( { 'tsconfig.json': JSON.stringify( {references: [{path: 'a'}], compilerOptions: {disableSourceOfProjectReferenceRedirect: true}, include: ['src']}, @@ -276,4 +290,5 @@ test('handles disableSourceOfProjectReferenceRedirect', (t) => { t.deepEqual(dependencyResolver.getDependencies(root + 'src/a.ts'), new Map([ ['../a/dist/file', [root + 'a/dist/file.d.ts']], ])); + assertAllDependenciesInProgram(dependencyResolver, program, t); }); From 026c5b542baef06b3e67bc9e3f1832ceb3940341 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Tue, 9 Feb 2021 22:16:44 +0100 Subject: [PATCH 26/37] add initial tests for program state --- .../wotan/test/program-state.spec.ts.md | 162 ++++++++ .../wotan/test/program-state.spec.ts.snap | Bin 0 -> 492 bytes packages/wotan/src/services/program-state.ts | 9 +- packages/wotan/test/program-state.spec.ts | 367 ++++++++++++++++++ 4 files changed, 535 insertions(+), 3 deletions(-) create mode 100644 baselines/packages/wotan/test/program-state.spec.ts.md create mode 100644 baselines/packages/wotan/test/program-state.spec.ts.snap create mode 100644 packages/wotan/test/program-state.spec.ts diff --git a/baselines/packages/wotan/test/program-state.spec.ts.md b/baselines/packages/wotan/test/program-state.spec.ts.md new file mode 100644 index 000000000..7b93a932a --- /dev/null +++ b/baselines/packages/wotan/test/program-state.spec.ts.md @@ -0,0 +1,162 @@ +# Snapshot report for `packages/wotan/test/program-state.spec.ts` + +The actual snapshot is saved in `program-state.spec.ts.snap`. + +Generated by [AVA](https://avajs.dev). + +## saves old state + +> Snapshot 1 + + `cs: false␊ + files:␊ + - config: '1234'␊ + hash: '-2704852577'␊ + result: []␊ + - config: '1234'␊ + dependencies:␊ + ./c:␊ + - 0␊ + hash: '-5185547329'␊ + result: []␊ + - dependencies:␊ + ./c:␊ + - 0␊ + hash: '-5185547329'␊ + - dependencies:␊ + ./b:␊ + - 1␊ + ./d:␊ + - 2␊ + hash: '-2126001415'␊ + global: []␊ + lookup:␊ + a.ts: 3␊ + b.ts: 1␊ + c.ts: 0␊ + d.ts: 2␊ + options: '-1350339532'␊ + v: 1␊ + ` + +> Snapshot 2 + + `cs: false␊ + files:␊ + - config: '1234'␊ + hash: '-2704852577'␊ + result: []␊ + - config: '1234'␊ + dependencies:␊ + ./c:␊ + - 0␊ + hash: '-5185547329'␊ + result: []␊ + - dependencies:␊ + "\\0":␊ + - 3␊ + hash: '5381'␊ + - dependencies:␊ + ./e:␊ + - 2␊ + - 3␊ + hash: '910822549'␊ + - dependencies:␊ + ./b:␊ + - 1␊ + ./d:␊ + - 3␊ + hash: '-2126001415'␊ + global: []␊ + lookup:␊ + a.ts: 4␊ + b.ts: 1␊ + c.ts: 0␊ + d.ts: 3␊ + e.ts: 2␊ + options: '-1350339532'␊ + v: 1␊ + ` + +> Snapshot 3 + + `cs: false␊ + files:␊ + - hash: '-2704852577'␊ + - dependencies:␊ + ./c:␊ + - 0␊ + hash: '-5185547329'␊ + - dependencies:␊ + "\\0":␊ + - 3␊ + e: null␊ + hash: '8844149038'␊ + - dependencies:␊ + ./e:␊ + - 2␊ + - 3␊ + hash: '910822549'␊ + - dependencies:␊ + ./b:␊ + - 1␊ + ./d:␊ + - 3␊ + hash: '-2126001415'␊ + global:␊ + - 2␊ + lookup:␊ + a.ts: 4␊ + b.ts: 1␊ + c.ts: 0␊ + d.ts: 3␊ + e.ts: 2␊ + options: '-1350339532'␊ + v: 1␊ + ` + +## doesn't discard results from old state + +> Snapshot 1 + + `cs: false␊ + files:␊ + - config: '1234'␊ + hash: '-3360789062'␊ + result: []␊ + - config: '1234'␊ + hash: '574235295'␊ + result: []␊ + - dependencies:␊ + ./c:␊ + - 1␊ + hash: '-5185547329'␊ + global:␊ + - 1␊ + lookup:␊ + a.ts: 0␊ + b.ts: 2␊ + c.ts: 1␊ + options: '5864093'␊ + v: 1␊ + ` + +> Snapshot 2 + + `cs: false␊ + files:␊ + - hash: '-3360789062'␊ + - hash: '4830905933'␊ + - dependencies:␊ + ./c:␊ + - 1␊ + hash: '-5185547329'␊ + global:␊ + - 1␊ + lookup:␊ + a.ts: 0␊ + b.ts: 2␊ + c.ts: 1␊ + options: '5864093'␊ + v: 1␊ + ` diff --git a/baselines/packages/wotan/test/program-state.spec.ts.snap b/baselines/packages/wotan/test/program-state.spec.ts.snap new file mode 100644 index 0000000000000000000000000000000000000000..081e9680c3c139b41cdabf14eecf126de28bf945 GIT binary patch literal 492 zcmV7?_g-08dfC0RfSWl?iEe{xjGUT#LGt8!|0E{pV;$t3X* z$IkZQ^l|md^X{r!*SGfM}K!+On6tW(1e!?kW(-)f~!JROO%dM{&f=JYaZ zgzb(O29Y)<%bX}MjSa02t>+x2t|xsLeVt1niU=KzBYzRYt2#|CXCig9mlrD-A$(Z0jCJx^r&PNj0E#0rOKTBty4th-ANIJK8 i7#jy&-yiHJCFhhY0`f1hG!h@Y2LJ$qLf@VM literal 0 HcmV?d00001 diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 697b0be06..fae0b46b2 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -291,7 +291,7 @@ class ProgramStateImpl implements ProgramState { private aggregate(): StaticProgramState { const oldState = this.tryReuseOldState(); const lookup: Record = {}; - const mapToIndex = ({fileName}: {fileName: string}) => lookup[this.relativePathNames.get(fileName)!]; + const mapToIndex = ({fileName}: {fileName: string}) => lookup[this.host.getCanonicalFileName(this.relativePathNames.get(fileName)!)]; const mapDependencies = (dependencies: ReadonlyMap) => { if (dependencies.size === 0) return; @@ -305,7 +305,7 @@ class ProgramStateImpl implements ProgramState { const files: StaticProgramState.FileState[] = []; const sourceFiles = this.program.getSourceFiles(); for (let i = 0; i < sourceFiles.length; ++i) - lookup[this.getRelativePath(sourceFiles[i].fileName)] = i; + lookup[this.host.getCanonicalFileName(this.getRelativePath(sourceFiles[i].fileName))] = i; for (const file of sourceFiles) { let results = this.fileResults.get(file.fileName); if (results === undefined && oldState !== undefined) { @@ -344,7 +344,10 @@ class ProgramStateImpl implements ProgramState { } private lookupFileIndex(fileName: string, oldState: StaticProgramState): number | undefined { - return oldState.lookup[this.host.getCanonicalFileName(this.getRelativePath(fileName))]; + fileName = this.host.getCanonicalFileName(this.getRelativePath(fileName)); + if (!oldState.cs && this.host.useCaseSensitiveFileNames()) + fileName = fileName.toLowerCase(); + return oldState.lookup[fileName]; } private remapFileNames(oldState: StaticProgramState): StaticProgramState { diff --git a/packages/wotan/test/program-state.spec.ts b/packages/wotan/test/program-state.spec.ts new file mode 100644 index 000000000..d5cb24928 --- /dev/null +++ b/packages/wotan/test/program-state.spec.ts @@ -0,0 +1,367 @@ +import 'reflect-metadata'; +import { DirectoryJSON, Volume } from 'memfs'; +import * as path from 'path'; +import { mapDefined, unixifyPath } from '../src/utils'; +import * as ts from 'typescript'; +import { Dirent } from 'fs'; +import { ProgramStateFactory, ProgramStateHost } from '../src/services/program-state'; +import { DependencyResolverFactory, DependencyResolverHost } from '../src/services/dependency-resolver'; +import { StatePersistence, StaticProgramState } from '@fimbul/ymir'; +import test from 'ava'; +import * as yaml from 'js-yaml'; + +function identity(v: T) { + return v; +} + +function setup(fileContents: DirectoryJSON, options: { subdir?: string, initialState?: StaticProgramState, caseSensitive?: boolean} = {}) { + const {root} = path.parse(unixifyPath(process.cwd())); + const cwd = options.subdir ? unixifyPath(path.join(root, options.subdir)) + '/' : root; + const vol = Volume.fromJSON(fileContents, cwd); + function fileExists(f: string) { + try { + return vol.statSync(f).isFile(); + } catch { + return false; + } + } + function readFile(f: string) { + try { + return vol.readFileSync(f, {encoding: 'utf8'}).toString(); + } catch { + return; + } + } + function readDirectory( + rootDir: string, + extensions: readonly string[], + excludes: readonly string[] | undefined, + includes: readonly string[], + depth?: number, + ) { + return ts.matchFiles( + rootDir, + extensions, + excludes, + includes, + true, + cwd, + depth, + (dir) => { + const files: string[] = []; + const directories: string[] = []; + const result: ts.FileSystemEntries = {files, directories}; + let entries; + try { + entries = vol.readdirSync(dir, {withFileTypes: true, encoding: 'utf8'}); + } catch { + return result; + } + for (const entry of entries) + (entry.isFile() ? files : directories).push(entry.name); + return result; + }, + identity, + ); + } + + const tsconfigPath = cwd + 'tsconfig.json'; + const commandLine = ts.getParsedCommandLineOfConfigFile(tsconfigPath, {}, { + fileExists, + readFile, + readDirectory, + getCurrentDirectory() { return cwd; }, + useCaseSensitiveFileNames: !!options.caseSensitive, + onUnRecoverableConfigFileDiagnostic(d) { + throw new Error(d.messageText); + }, + })!; + const resolutionCache = ts.createModuleResolutionCache(cwd, identity, commandLine.options); + const compilerHost: ts.CompilerHost & DependencyResolverHost & ProgramStateHost = { + fileExists, + readFile, + readDirectory, + directoryExists(d) { + try { + return vol.statSync(d).isDirectory(); + } catch { + return false; + } + }, + getCanonicalFileName: options.caseSensitive ? identity : (f) => f.toLowerCase(), + getCurrentDirectory() { + return cwd; + }, + getNewLine() { + return '\n'; + }, + useCaseSensitiveFileNames() { + return !!options.caseSensitive; + }, + getDefaultLibFileName: ts.getDefaultLibFilePath, + getSourceFile(fileName, languageVersion) { + const content = readFile(fileName); + return content === undefined ? undefined : ts.createSourceFile(fileName, content, languageVersion, true); + }, + writeFile() { + throw new Error('not implemented'); + }, + realpath: identity, + getDirectories(d) { + return mapDefined(vol.readdirSync(d, {encoding: 'utf8'}), (f) => { + return this.directoryExists!(d + '/' + f) ? f : undefined; + }); + }, + resolveModuleNames(modules, containingFile, _, redirectedReference, o) { + return modules.map( + (m) => ts.resolveModuleName(m, containingFile, o, this, resolutionCache, redirectedReference).resolvedModule); + }, + }; + const program = ts.createProgram({ + host: compilerHost, + options: commandLine.options, + rootNames: commandLine.fileNames, + projectReferences: commandLine.projectReferences, + }); + const persistence: StatePersistence = { + loadState() { return options.initialState; }, + saveState() {}, + }; + const factory = new ProgramStateFactory(new DependencyResolverFactory(), persistence) + const programState = factory.create(program, compilerHost, tsconfigPath); + return {vol, compilerHost, program, programState, cwd, persistence, tsconfigPath, factory}; +} + +function generateOldState(configHash: string, ...args: Parameters) { + const {program, programState, persistence} = setup(...args); + for (const file of program.getSourceFiles()) + programState.setFileResult(file.fileName, configHash, []); + let savedState!: StaticProgramState; + persistence.saveState = (_, state) => savedState = state; + programState.save(); + return savedState; +} + +test('saves old state', (t) => { + let {programState, persistence, tsconfigPath, cwd, program, vol, compilerHost, factory} = setup({ + 'tsconfig.json': JSON.stringify({compilerOptions: {strict: true}}), + 'a.ts': 'import "./b"; import "./d";', + 'b.ts': 'import "./c";', + 'c.ts': 'export {}', + 'd.ts': 'export {}', + }, {subdir: 'foo'}); + let savedState: StaticProgramState | undefined; + persistence.saveState = (project, state) => { + t.is(project, tsconfigPath); + const {ts: tsVersion, ...rest} = state; + t.is(tsVersion, ts.version); + t.snapshot(yaml.dump(rest, {sortKeys: true})); + savedState = state; + }; + persistence.loadState = (project) => { + t.is(project, tsconfigPath); + return savedState; + } + t.is(programState.getUpToDateResult(cwd + 'a.ts', ''), undefined); + + programState.setFileResult(cwd + 'a.ts', '1234', []); + programState.setFileResult(cwd + 'b.ts', '1234', []); + programState.setFileResult(cwd + 'c.ts', '1234', []); + programState.setFileResult(cwd + 'd.ts', '1234', []); + + vol.writeFileSync(cwd + 'd.ts', 'import "./c";'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'd.ts'); + + programState.save(); + t.not(savedState, undefined); + + programState = factory.create(program, compilerHost, tsconfigPath); + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + t.is(programState.getUpToDateResult(cwd + 'd.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '5678'), undefined); + + vol.writeFileSync(cwd + 'd.ts', 'import "./e"; declare module "./e" {}'); + vol.writeFileSync(cwd + 'e.ts', ''); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'd.ts'); + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + t.is(programState.getUpToDateResult(cwd + 'd.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'e.ts', '1234'), undefined); + + programState.save(); + + vol.writeFileSync(cwd + 'e.ts', 'var v: import("e");'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'e.ts'); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'd.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'e.ts', '1234'), undefined); + + savedState = undefined; + programState.save(); + t.not(savedState, undefined); +}); + +test('can start with a warm cache', (t) => { + const state = generateOldState( + '1234', + { + 'tsconfig.json': '{}', + 'A.ts': '', + 'b.ts': '', + 'C.ts': '', + 'd.ts': '', + }, + ); + const {programState, cwd} = setup( + { + 'tsconfig.json': '{}', + 'A.ts': '', + 'b.ts': '', + 'c.ts': '', + 'D.ts': '', + }, + { + subdir: 'foo', + initialState: state, + }, + ); + t.deepEqual(programState.getUpToDateResult(cwd + 'A.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'D.ts', '1234'), []); +}); + +test('can handle case-sensitive old state on case-insensitive system', (t) => { + const state = generateOldState( + '1234', + { + 'tsconfig.json': '{}', + 'A.ts': '', + 'b.ts': '', + }, + { + caseSensitive: true, + }, + ); + const {programState, cwd} = setup( + { + 'tsconfig.json': '{}', + 'a.ts': '', + 'b.ts': '', + }, + { + caseSensitive: false, + initialState: state, + }, + ); + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); +}); + +test('can handle case-insensitive old state on case-sensitive system', (t) => { + const state = generateOldState( + '1234', + { + 'tsconfig.json': '{}', + 'a.ts': '', + 'b.ts': '', + }, + { + caseSensitive: false, + }, + ); + const {programState, cwd} = setup( + { + 'tsconfig.json': '{}', + 'A.ts': '', + 'b.ts': '', + }, + { + caseSensitive: true, + initialState: state, + }, + ); + t.deepEqual(programState.getUpToDateResult(cwd + 'A.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); +}); + +test("doesn't discard results from old state", (t) => { + const state = generateOldState( + '1234', + { + 'tsconfig.json': '{}', + 'a.ts': 'export {};', + 'b.ts': 'export {};', + 'c.ts': 'var v;', + }, + ); + let {programState, cwd, persistence, vol, program, compilerHost} = setup( + { + 'tsconfig.json': '{}', + 'a.ts': 'export {};', + 'b.ts': 'export {};', + 'c.ts': 'var v;', + }, + { + initialState: state, + }, + ); + + persistence.saveState = (_, s) => t.deepEqual(s, state); + programState.save(); + + vol.writeFileSync(cwd + 'b.ts', 'import "./c";'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'b.ts'); + + persistence.saveState = (_, {ts: _ts, ...rest}) => t.snapshot(yaml.dump(rest, {sortKeys: true})); + programState.save(); + + vol.writeFileSync(cwd + 'c.ts', 'var v = 1;'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'c.ts'); + + programState.save(); +}); + +test('uses relative paths for compilerOptions', (t) => { + const files = { + 'tsconfig.json': JSON.stringify({compilerOptions: {outDir: './dist', rootDirs: ['./src', './lib']}}), + 'a.ts': '', + }; + t.deepEqual(generateOldState('', files, {subdir: 'a'}), generateOldState('', files, {subdir: 'b/c'})); +}); From 4ba0d63c5658b3870fdad23773afe9a2a529e78b Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Wed, 10 Feb 2021 19:20:55 +0100 Subject: [PATCH 27/37] find global augmentation in ambient module --- packages/wotan/src/services/dependency-resolver.ts | 10 ++++++++++ packages/wotan/src/services/program-state.ts | 3 ++- packages/wotan/test/dependency-resolver.spec.ts | 8 ++++++++ packages/wotan/test/program-state.spec.ts | 4 ++-- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/wotan/src/services/dependency-resolver.ts b/packages/wotan/src/services/dependency-resolver.ts index eb69024df..78cf25597 100644 --- a/packages/wotan/src/services/dependency-resolver.ts +++ b/packages/wotan/src/services/dependency-resolver.ts @@ -241,6 +241,16 @@ function collectFileMetadata(sourceFile: ts.SourceFile): MetaData { affectsGlobalScope = true; } else if (isModuleDeclaration(statement) && statement.name.kind === ts.SyntaxKind.StringLiteral) { ambientModules.add(statement.name.text); + if (!isExternalModule && !affectsGlobalScope && statement.body !== undefined) { + // search for global augmentations in ambient module blocks + for (const s of (statement.body).statements) { + if (s.flags & ts.NodeFlags.GlobalAugmentation) { + affectsGlobalScope = true; + break; + } + } + } + } else if (isNamespaceExportDeclaration(statement)) { affectsGlobalScope = true; } else if (affectsGlobalScope === undefined) { // files that only consist of ambient modules do not affect global scope diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index fae0b46b2..52a110a76 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -291,7 +291,8 @@ class ProgramStateImpl implements ProgramState { private aggregate(): StaticProgramState { const oldState = this.tryReuseOldState(); const lookup: Record = {}; - const mapToIndex = ({fileName}: {fileName: string}) => lookup[this.host.getCanonicalFileName(this.relativePathNames.get(fileName)!)]; + const mapToIndex = ({fileName}: {fileName: string}) => + lookup[this.host.getCanonicalFileName(this.relativePathNames.get(fileName)!)]; const mapDependencies = (dependencies: ReadonlyMap) => { if (dependencies.size === 0) return; diff --git a/packages/wotan/test/dependency-resolver.spec.ts b/packages/wotan/test/dependency-resolver.spec.ts index aa903472e..23d98c03b 100644 --- a/packages/wotan/test/dependency-resolver.spec.ts +++ b/packages/wotan/test/dependency-resolver.spec.ts @@ -235,6 +235,14 @@ test('resolves imports', (t) => { assertAllDependenciesInProgram(dependencyResolver, program, t); }); +test('finds global augmentations in ambient modules', (t) => { + const {dependencyResolver, root} = setup({ + 'tsconfig.json': '{}', + 'a.ts': 'declare module "foo" { global { var v: number; } }', + }); + t.deepEqual(dependencyResolver.getFilesAffectingGlobalScope(), [root + 'a.ts']); +}); + test('handles useSourceOfProjectReferenceRedirect', (t) => { const {dependencyResolver, program, root} = setup( { diff --git a/packages/wotan/test/program-state.spec.ts b/packages/wotan/test/program-state.spec.ts index d5cb24928..772c7db05 100644 --- a/packages/wotan/test/program-state.spec.ts +++ b/packages/wotan/test/program-state.spec.ts @@ -127,7 +127,7 @@ function setup(fileContents: DirectoryJSON, options: { subdir?: string, initialS loadState() { return options.initialState; }, saveState() {}, }; - const factory = new ProgramStateFactory(new DependencyResolverFactory(), persistence) + const factory = new ProgramStateFactory(new DependencyResolverFactory(), persistence); const programState = factory.create(program, compilerHost, tsconfigPath); return {vol, compilerHost, program, programState, cwd, persistence, tsconfigPath, factory}; } @@ -161,7 +161,7 @@ test('saves old state', (t) => { persistence.loadState = (project) => { t.is(project, tsconfigPath); return savedState; - } + }; t.is(programState.getUpToDateResult(cwd + 'a.ts', ''), undefined); programState.setFileResult(cwd + 'a.ts', '1234', []); From 1ebf6d67c075d5dabf8612673444446c69e7259c Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Wed, 10 Feb 2021 20:42:05 +0100 Subject: [PATCH 28/37] more tests for program state --- .../wotan/test/program-state.spec.ts.md | 48 +++++++++++ .../wotan/test/program-state.spec.ts.snap | Bin 492 -> 615 bytes packages/wotan/test/program-state.spec.ts | 76 +++++++++++++++++- 3 files changed, 123 insertions(+), 1 deletion(-) diff --git a/baselines/packages/wotan/test/program-state.spec.ts.md b/baselines/packages/wotan/test/program-state.spec.ts.md index 7b93a932a..0b026a58b 100644 --- a/baselines/packages/wotan/test/program-state.spec.ts.md +++ b/baselines/packages/wotan/test/program-state.spec.ts.md @@ -160,3 +160,51 @@ Generated by [AVA](https://avajs.dev). options: '5864093'␊ v: 1␊ ` + +## handles circular dependencies + +> Snapshot 1 + + `cs: false␊ + files:␊ + - config: '1234'␊ + hash: '-3360789062'␊ + result: []␊ + - dependencies:␊ + ./b:␊ + - 3␊ + ./e:␊ + - 0␊ + hash: '-2126000326'␊ + - dependencies:␊ + ./d:␊ + - 1␊ + hash: '-2335134369'␊ + - dependencies:␊ + ./a:␊ + - 4␊ + ./c:␊ + - 2␊ + ./d:␊ + - 1␊ + hash: '-3138187278'␊ + - dependencies:␊ + ./b:␊ + - 3␊ + ./c:␊ + - 2␊ + ./d:␊ + - 1␊ + ./e:␊ + - 0␊ + hash: '-2424661969'␊ + global: []␊ + lookup:␊ + a.ts: 4␊ + b.ts: 3␊ + c.ts: 2␊ + d.ts: 1␊ + e.ts: 0␊ + options: '5864093'␊ + v: 1␊ + ` diff --git a/baselines/packages/wotan/test/program-state.spec.ts.snap b/baselines/packages/wotan/test/program-state.spec.ts.snap index 081e9680c3c139b41cdabf14eecf126de28bf945..0d9ec312f57600be39b935e00b59f9ee5f7f2904 100644 GIT binary patch literal 615 zcmV-t0+{_lRzV%DosrhRynzTFa+x36ot>UWEE>l{MQp1i!ezkV2o-_{?0{`&oW zH&>5(#rm>TtiAIp&Qk8T(?gduOY>647`jm!=)*7yQb#w8zTqYDFx+qqlTcu0CK<-2 zn{H-8AHc%4kl3Py{vb#v(bzHWtmP;DU=+lD5PMOWjqyz5^7rOgjMJ1Q=Md$$?vSz)IPx%Ct>5ARvgasV$nq&8b8CaKEY=qs+31R7r0{ z$-s?rFj11+pN!NocV(RMR*4RBOWncCEfvwsPEWy+tsvT30 z=wjy&Kce|3xy>kWD)|DB&znz;6TvyNfu7V$FLwQ0GF+C7mL((Y%5#8f%Tr&AQfC6N z#9RV2J*dB(+lz(KQyVkEDYxym=v!ot=VpGoSJLFKRd1hkt3TV@(=(1R!W+94N<5^* zgUfEE6;OQw90N!Wv<}wMe*$ayE?%A;q{kB?o5M|*g1T8d<3$oIDCO+8iNk4#vqs`j z&gTW#zpH0o5BiLHq@G)rjAO);D60Asf-%A*hQ{CNzs6E8?r}Cae*iAP!?Gj_006lo BDQ5rx literal 492 zcmV7?_g-08dfC0RfSWl?iEe{xjGUT#LGt8!|0E{pV;$t3X* z$IkZQ^l|md^X{r!*SGfM}K!+On6tW(1e!?kW(-)f~!JROO%dM{&f=JYaZ zgzb(O29Y)<%bX}MjSa02t>+x2t|xsLeVt1niU=KzBYzRYt2#|CXCig9mlrD-A$(Z0jCJx^r&PNj0E#0rOKTBty4th-ANIJK8 i7#jy&-yiHJCFhhY0`f1hG!h@Y2LJ$qLf@VM diff --git a/packages/wotan/test/program-state.spec.ts b/packages/wotan/test/program-state.spec.ts index 772c7db05..3388294e3 100644 --- a/packages/wotan/test/program-state.spec.ts +++ b/packages/wotan/test/program-state.spec.ts @@ -363,5 +363,79 @@ test('uses relative paths for compilerOptions', (t) => { 'tsconfig.json': JSON.stringify({compilerOptions: {outDir: './dist', rootDirs: ['./src', './lib']}}), 'a.ts': '', }; - t.deepEqual(generateOldState('', files, {subdir: 'a'}), generateOldState('', files, {subdir: 'b/c'})); + const state = generateOldState('', files, {subdir: 'a'}); + t.deepEqual(generateOldState('', files, {subdir: 'b/c'}), state); + + t.notDeepEqual( + generateOldState('', { + 'tsconfig.json': JSON.stringify({compilerOptions: {outDir: './out', rootDirs: ['./src', './lib']}}), + 'a.ts': '', + }, {subdir: 'a'}), + state, + ); +}); + +test('handles assumeChangesOnlyAffectDirectDependencies', (t) => { + const state = generateOldState('1234', { + 'tsconfig.json': JSON.stringify({compilerOptions: {assumeChangesOnlyAffectDirectDependencies: true}}), + 'a.ts': 'import "./b";', + 'b.ts': 'import "./c";', + 'c.ts': 'export {};', + 'global.ts': 'declare global {}; import "./b";', + }); + + const {programState, cwd} = setup( + { + 'tsconfig.json': JSON.stringify({compilerOptions: {assumeChangesOnlyAffectDirectDependencies: true}}), + 'a.ts': 'import "./b";', + 'b.ts': 'import "./c";', + 'c.ts': 'export {}; foo;', // <-- change here + 'global.ts': 'declare global {}; import "./b";', + }, + { + initialState: state, + }, + ); + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); + t.deepEqual(programState.getUpToDateResult(cwd + 'global.ts', '1234'), []); +}); + +test('handles circular dependencies', (t) => { + const files = { + 'tsconfig.json': '{}', + 'a.ts': 'import "./b"; import "./c"; import "./d"; import "./e";', + 'b.ts': 'import "./a"; import "./c"; import "./d";', + 'c.ts': 'import "./d";', + 'd.ts': 'import "./b"; import "./e";', + 'e.ts': 'export {};', + }; + const state = generateOldState('1234', files); + + let {programState, cwd, vol, program, compilerHost, persistence} = setup(files, {initialState: state}); + + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'd.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'e.ts', '1234'), []); + + vol.appendFileSync(cwd + 'c.ts', 'foo;'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'c.ts'); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'd.ts', '1234'), undefined); + t.deepEqual(programState.getUpToDateResult(cwd + 'e.ts', '1234'), []); + + persistence.saveState = (_, {ts: _ts, ...rest}) => t.snapshot(yaml.dump(rest, {sortKeys: true})); + programState.save(); }); From 96b0a80f997d403be2d7d580ff31847e85e26066 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Wed, 10 Feb 2021 21:36:08 +0100 Subject: [PATCH 29/37] more testing with circular dependencies --- .../wotan/test/program-state.spec.ts.md | 5 + .../wotan/test/program-state.spec.ts.snap | Bin 615 -> 630 bytes packages/wotan/test/program-state.spec.ts | 98 ++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/baselines/packages/wotan/test/program-state.spec.ts.md b/baselines/packages/wotan/test/program-state.spec.ts.md index 0b026a58b..5fd564c7e 100644 --- a/baselines/packages/wotan/test/program-state.spec.ts.md +++ b/baselines/packages/wotan/test/program-state.spec.ts.md @@ -198,6 +198,10 @@ Generated by [AVA](https://avajs.dev). ./e:␊ - 0␊ hash: '-2424661969'␊ + - dependencies:␊ + ./a:␊ + - 4␊ + hash: '-5185549507'␊ global: []␊ lookup:␊ a.ts: 4␊ @@ -205,6 +209,7 @@ Generated by [AVA](https://avajs.dev). c.ts: 2␊ d.ts: 1␊ e.ts: 0␊ + root.ts: 5␊ options: '5864093'␊ v: 1␊ ` diff --git a/baselines/packages/wotan/test/program-state.spec.ts.snap b/baselines/packages/wotan/test/program-state.spec.ts.snap index 0d9ec312f57600be39b935e00b59f9ee5f7f2904..3aecba8c9b853b13f78379a84c8f818f2e161cfe 100644 GIT binary patch literal 630 zcmV-+0*U=WRzV_VS&LEBu^=K_wMXF^9%S5&V!W)Cxog>hzW^5zy=bC z1^5FM8%(GJGb{|O_|D4V5XU%$3WTMTUOwG@@80*ldr1t#=ovT8emHu0_3_2aA9oH8 zpFc=DhWs9O4P$tF<@Jk=?)!WBn;m}P(Uq=3pY_VMf2vsbpI&;kc@)QAH}^h$`Sz}- z(Ce$^a?g}&)s%ToE8fnEalX6P4RX&K`bi#{!#Ig@&$O(*6=dl!-tsIP6QFhhS=P3n zZx=uxz(Fn-$RQ=Z8|9N^>{&NA)X6X!MQIqNL0odlZ~fX}8pIHHNbuM$uV2UVD~Ne37><}Pu0Q@RB@v`_ad)hMBkgT>NCPYD?2 zj8VamYnxk1Ht-YWY?5VnCL?*wUmq6@BM%K!B#Q`CB<~%nNM1Ekk-Xh*mW|a9W@e*t zoTWu*Gwx6%z&7v7?K3TTuJeama`8##QUZ(!wnQrji+i9H&jlj1cxJilXZ?`Yk7)gH zJ9!R`T6yZV&^YCQ1Y!c9sWJX`ZBLfAp16o|MwsiiwZ28=Xkq2Y>yjqER+Hk#+Yjd_ z#q7ky6tl*s3)&v6?LqqKl8RA%3=9DX2Bglc&;N;8Yq!wa{8*k&h+?WY*%Z}H?l~5L z5htXWtH0|w9LI6m=s4(My@dB~>%Ffx`h?y{y>CS^o|R6Lq{_!RrI-o?jepMnnwMsI QkCU%DosrhRynzTFa+x36ot>UWEE>l{MQp1i!ezkV2o-_{?0{`&oW zH&>5(#rm>TtiAIp&Qk8T(?gduOY>647`jm!=)*7yQb#w8zTqYDFx+qqlTcu0CK<-2 zn{H-8AHc%4kl3Py{vb#v(bzHWtmP;DU=+lD5PMOWjqyz5^7rOgjMJ1Q=Md$$?vSz)IPx%Ct>5ARvgasV$nq&8b8CaKEY=qs+31R7r0{ z$-s?rFj11+pN!NocV(RMR*4RBOWncCEfvwsPEWy+tsvT30 z=wjy&Kce|3xy>kWD)|DB&znz;6TvyNfu7V$FLwQ0GF+C7mL((Y%5#8f%Tr&AQfC6N z#9RV2J*dB(+lz(KQyVkEDYxym=v!ot=VpGoSJLFKRd1hkt3TV@(=(1R!W+94N<5^* zgUfEE6;OQw90N!Wv<}wMe*$ayE?%A;q{kB?o5M|*g1T8d<3$oIDCO+8iNk4#vqs`j z&gTW#zpH0o5BiLHq@G)rjAO);D60Asf-%A*hQ{CNzs6E8?r}Cae*iAP!?Gj_006lo BDQ5rx diff --git a/packages/wotan/test/program-state.spec.ts b/packages/wotan/test/program-state.spec.ts index 3388294e3..87dbd3907 100644 --- a/packages/wotan/test/program-state.spec.ts +++ b/packages/wotan/test/program-state.spec.ts @@ -405,6 +405,7 @@ test('handles assumeChangesOnlyAffectDirectDependencies', (t) => { test('handles circular dependencies', (t) => { const files = { 'tsconfig.json': '{}', + 'root.ts': 'import "./a";', 'a.ts': 'import "./b"; import "./c"; import "./d"; import "./e";', 'b.ts': 'import "./a"; import "./c"; import "./d";', 'c.ts': 'import "./d";', @@ -415,6 +416,7 @@ test('handles circular dependencies', (t) => { let {programState, cwd, vol, program, compilerHost, persistence} = setup(files, {initialState: state}); + t.deepEqual(programState.getUpToDateResult(cwd + 'root.ts', '1234'), []); t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); @@ -430,6 +432,7 @@ test('handles circular dependencies', (t) => { }); programState.update(program, cwd + 'c.ts'); + t.is(programState.getUpToDateResult(cwd + 'root.ts', '1234'), undefined); t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); @@ -439,3 +442,98 @@ test('handles circular dependencies', (t) => { persistence.saveState = (_, {ts: _ts, ...rest}) => t.snapshot(yaml.dump(rest, {sortKeys: true})); programState.save(); }); + +test('handles multiple level of circular dependencies', (t) => { + const files = { + 'tsconfig.json': '{}', + 'a.ts': 'import "./b"; import "./c";', + 'b.ts': 'import "./a";', + 'c.ts': 'import "./d"; import "./e";', + 'd.ts': 'import "./c";', + 'e.ts': 'import "./f";', + 'f.ts': 'import "./d";', + }; + const state = generateOldState('1234', files); + + let {programState, cwd, vol, program, compilerHost} = setup(files, {initialState: state}); + + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'd.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'e.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'f.ts', '1234'), []); + + vol.appendFileSync(cwd + 'b.ts', 'foo;'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'b.ts'); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'd.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'e.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'f.ts', '1234'), []); + + ({programState, cwd, vol, program, compilerHost} = setup(files, {initialState: state})); + vol.appendFileSync(cwd + 'f.ts', 'foo;'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'f.ts'); + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'd.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'e.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'f.ts', '1234'), undefined); +}); + +test('merges multiple level of circular dependencies', (t) => { + const files = { + 'tsconfig.json': '{}', + 'a.ts': 'import "./b"; import "./c";', + 'b.ts': 'import "./a";', + 'c.ts': 'import "./d"; import "./e";', + 'd.ts': 'import "./c";', + 'e.ts': 'import "./f";', + 'f.ts': 'import "./b"; import "./g";', + 'g.ts': 'export {};', + }; + const state = generateOldState('1234', files); + + let {programState, cwd, vol, program, compilerHost} = setup(files, {initialState: state}); + + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'd.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'e.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'f.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'g.ts', '1234'), []); + + vol.appendFileSync(cwd + 'g.ts', 'foo;'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'g.ts'); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'd.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'e.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'f.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'g.ts', '1234'), undefined); +}); From a0b8b1425f907544e0d5561ae855121dc76cefec Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Wed, 10 Feb 2021 22:05:04 +0100 Subject: [PATCH 30/37] make tests work as intended --- .../wotan/test/program-state.spec.ts.md | 34 +++++++++--------- .../wotan/test/program-state.spec.ts.snap | Bin 630 -> 622 bytes packages/wotan/test/program-state.spec.ts | 16 ++++----- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/baselines/packages/wotan/test/program-state.spec.ts.md b/baselines/packages/wotan/test/program-state.spec.ts.md index 5fd564c7e..0aa83d06e 100644 --- a/baselines/packages/wotan/test/program-state.spec.ts.md +++ b/baselines/packages/wotan/test/program-state.spec.ts.md @@ -170,34 +170,34 @@ Generated by [AVA](https://avajs.dev). - config: '1234'␊ hash: '-3360789062'␊ result: []␊ - - dependencies:␊ - ./b:␊ - - 3␊ - ./e:␊ - - 0␊ - hash: '-2126000326'␊ - dependencies:␊ ./d:␊ - - 1␊ + - 3␊ hash: '-2335134369'␊ - dependencies:␊ ./a:␊ - 4␊ ./c:␊ - - 2␊ - ./d:␊ - 1␊ - hash: '-3138187278'␊ + ./d:␊ + - 3␊ + hash: '-10011708814'␊ - dependencies:␊ ./b:␊ - - 3␊ - ./c:␊ - 2␊ - ./d:␊ + ./e:␊ + - 0␊ + hash: '970287226'␊ + - dependencies:␊ + ./b:␊ + - 2␊ + ./c:␊ - 1␊ + ./d:␊ + - 3␊ ./e:␊ - 0␊ - hash: '-2424661969'␊ + hash: '3028017583'␊ - dependencies:␊ ./a:␊ - 4␊ @@ -205,9 +205,9 @@ Generated by [AVA](https://avajs.dev). global: []␊ lookup:␊ a.ts: 4␊ - b.ts: 3␊ - c.ts: 2␊ - d.ts: 1␊ + b.ts: 2␊ + c.ts: 1␊ + d.ts: 3␊ e.ts: 0␊ root.ts: 5␊ options: '5864093'␊ diff --git a/baselines/packages/wotan/test/program-state.spec.ts.snap b/baselines/packages/wotan/test/program-state.spec.ts.snap index 3aecba8c9b853b13f78379a84c8f818f2e161cfe..3432dcf609523a0963e8258cf7b95e3c6fbab8c6 100644 GIT binary patch literal 622 zcmV-!0+IbeRzVp$QdAQ3*FFLAK!Q+&q<02PW5DN-Tv12jlT zAR6EYB08v03TkL5Xz{KU;{%`NJBSciT50^`eLM5s>^MdU_0aXRAC6vLeSGoq$DK!q z&-aoJ($9Vuq0z0?*Dp4@@9$-AcI1VFD_w&=>lJHn1)+xzo?d#jc@#xoH}8M?^6g#E zpz2hy_?cp@So)mPhPShd&Uf#2gUqu>ew>BYD2l_(vuwL>2Wc{jwmjP*6qu7kw!Q6V z+d0q&aIugIyR^V}!)zK)Jp0Cm**OfyVKNMpAS$@@vwm%O9O_SChK~Yp0t{T`G!Xbz z0#?F76(((tBN!7Ru#|)~oj0g5&=L--Fs>3SfvFUfx=XOOucA2z$Wy>Xa0v@ac`Obm zl5?gw7LK(QrvpDWzQ$>Kdpg$J{PjuRE_&0zM7oB+L^|!EiFBr+iF9(iX*w}4+)Brj zC{6Ov;nHPT96f z7~zdq7fO3bX%8;1F0B~VC%`d);y~-n`uv}mwRRJ)&HY<&kfh0`ux_6J!lDS4l;(5w zcO8di9A}M={}|Q_c>lKE`+B2K=#JF;mKWpM$uy3ud{Qz-n8G0cB`>WakCU_VS&LEBu^=K_wMXF^9%S5&V!W)Cxog>hzW^5zy=bC z1^5FM8%(GJGb{|O_|D4V5XU%$3WTMTUOwG@@80*ldr1t#=ovT8emHu0_3_2aA9oH8 zpFc=DhWs9O4P$tF<@Jk=?)!WBn;m}P(Uq=3pY_VMf2vsbpI&;kc@)QAH}^h$`Sz}- z(Ce$^a?g}&)s%ToE8fnEalX6P4RX&K`bi#{!#Ig@&$O(*6=dl!-tsIP6QFhhS=P3n zZx=uxz(Fn-$RQ=Z8|9N^>{&NA)X6X!MQIqNL0odlZ~fX}8pIHHNbuM$uV2UVD~Ne37><}Pu0Q@RB@v`_ad)hMBkgT>NCPYD?2 zj8VamYnxk1Ht-YWY?5VnCL?*wUmq6@BM%K!B#Q`CB<~%nNM1Ekk-Xh*mW|a9W@e*t zoTWu*Gwx6%z&7v7?K3TTuJeama`8##QUZ(!wnQrji+i9H&jlj1cxJilXZ?`Yk7)gH zJ9!R`T6yZV&^YCQ1Y!c9sWJX`ZBLfAp16o|MwsiiwZ28=Xkq2Y>yjqER+Hk#+Yjd_ z#q7ky6tl*s3)&v6?LqqKl8RA%3=9DX2Bglc&;N;8Yq!wa{8*k&h+?WY*%Z}H?l~5L z5htXWtH0|w9LI6m=s4(My@dB~>%Ffx`h?y{y>CS^o|R6Lq{_!RrI-o?jepMnnwMsI QkCU { const files = { 'tsconfig.json': '{}', 'root.ts': 'import "./a";', - 'a.ts': 'import "./b"; import "./c"; import "./d"; import "./e";', - 'b.ts': 'import "./a"; import "./c"; import "./d";', + 'a.ts': 'import "./e"; import "./d"; import "./c"; import "./b";', + 'b.ts': 'import "./d"; import "./c"; import "./a";', 'c.ts': 'import "./d";', - 'd.ts': 'import "./b"; import "./e";', + 'd.ts': 'import "./e"; import "./b";', 'e.ts': 'export {};', }; const state = generateOldState('1234', files); @@ -446,9 +446,9 @@ test('handles circular dependencies', (t) => { test('handles multiple level of circular dependencies', (t) => { const files = { 'tsconfig.json': '{}', - 'a.ts': 'import "./b"; import "./c";', + 'a.ts': 'import "./c"; import "./b";', 'b.ts': 'import "./a";', - 'c.ts': 'import "./d"; import "./e";', + 'c.ts': 'import "./e"; import "./d";', 'd.ts': 'import "./c";', 'e.ts': 'import "./f";', 'f.ts': 'import "./d";', @@ -500,12 +500,12 @@ test('handles multiple level of circular dependencies', (t) => { test('merges multiple level of circular dependencies', (t) => { const files = { 'tsconfig.json': '{}', - 'a.ts': 'import "./b"; import "./c";', + 'a.ts': 'import "./c"; import "./b";', 'b.ts': 'import "./a";', - 'c.ts': 'import "./d"; import "./e";', + 'c.ts': 'import "./e"; import "./d";', 'd.ts': 'import "./c";', 'e.ts': 'import "./f";', - 'f.ts': 'import "./b"; import "./g";', + 'f.ts': 'import "./g"; import "./b";', 'g.ts': 'export {};', }; const state = generateOldState('1234', files); From e80c55690fb815525322e6cdca7806ff71516c6c Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 11 Feb 2021 19:26:14 +0100 Subject: [PATCH 31/37] more circularities --- packages/wotan/test/program-state.spec.ts | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/packages/wotan/test/program-state.spec.ts b/packages/wotan/test/program-state.spec.ts index dc6acbe69..2cc4ceb2e 100644 --- a/packages/wotan/test/program-state.spec.ts +++ b/packages/wotan/test/program-state.spec.ts @@ -537,3 +537,52 @@ test('merges multiple level of circular dependencies', (t) => { t.is(programState.getUpToDateResult(cwd + 'f.ts', '1234'), undefined); t.is(programState.getUpToDateResult(cwd + 'g.ts', '1234'), undefined); }); + +test('merges multiple level of circular dependencies II', (t) => { + const files = { + 'tsconfig.json': '{}', + 'a.ts': 'import "./e"; import "./b"; import "./a1";', + 'a1.ts': 'import "./a";', + 'b.ts': 'import "./b1";', + 'b1.ts': 'import "./c"; import "./b2"; import "./b";', + 'c.ts': 'import "./d";', + 'd.ts': 'import "./d2"; import "./d1";', + 'd1.ts': 'import "./d";', + 'd2.ts': 'import "./a";', + 'e.ts': 'export {};', + }; + const state = generateOldState('1234', files); + + let {programState, cwd, vol, program, compilerHost} = setup(files, {initialState: state}); + + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'a1.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b1.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'd.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'd1.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'd2.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'e.ts', '1234'), []); + + vol.appendFileSync(cwd + 'e.ts', 'foo;'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'e.ts'); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'a1.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b1.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'd.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'd1.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'd2.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'e.ts', '1234'), undefined); +}); + + From 5cda856ad19b34cd33ac6049d44f314055d0a71f Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 11 Feb 2021 20:25:55 +0100 Subject: [PATCH 32/37] program state tested, yeah --- packages/wotan/src/services/program-state.ts | 4 +- packages/wotan/test/program-state.spec.ts | 136 ++++++++++++++++++- 2 files changed, 135 insertions(+), 5 deletions(-) diff --git a/packages/wotan/src/services/program-state.ts b/packages/wotan/src/services/program-state.ts index 52a110a76..ab160e1e6 100644 --- a/packages/wotan/src/services/program-state.ts +++ b/packages/wotan/src/services/program-state.ts @@ -218,8 +218,6 @@ class ProgramStateImpl implements ProgramState { return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); for (const key of keys) { let newDeps = dependencies.get(key); - if (newDeps === undefined) - return markAsOutdated(parents, index, cycles, this.dependenciesUpToDate); const oldDeps = old.dependencies![key]; if (oldDeps === null) { if (newDeps !== null) @@ -426,7 +424,7 @@ function markAsOutdated(parents: readonly number[], index: number, cycles: Reado } function compareHashKey(a: {hash: string}, b: {hash: string}) { - return a.hash < b.hash ? -1 : a.hash === b.hash ? 0 : 1; + return +(a.hash >= b.hash) - +(a.hash <= b.hash); } const enum CompilerOptionKind { diff --git a/packages/wotan/test/program-state.spec.ts b/packages/wotan/test/program-state.spec.ts index 2cc4ceb2e..0c8b26b9c 100644 --- a/packages/wotan/test/program-state.spec.ts +++ b/packages/wotan/test/program-state.spec.ts @@ -14,10 +14,17 @@ function identity(v: T) { return v; } -function setup(fileContents: DirectoryJSON, options: { subdir?: string, initialState?: StaticProgramState, caseSensitive?: boolean} = {}) { +function setup( + fileContents: DirectoryJSON, + options: {subdir?: string, initialState?: StaticProgramState, caseSensitive?: boolean, symlinks?: ReadonlyArray<[string, string]>} = {}, +) { const {root} = path.parse(unixifyPath(process.cwd())); const cwd = options.subdir ? unixifyPath(path.join(root, options.subdir)) + '/' : root; const vol = Volume.fromJSON(fileContents, cwd); + if (options.symlinks) + for (const [from, to] of options.symlinks) + vol.symlinkSync(cwd + to, cwd + from); + function fileExists(f: string) { try { return vol.statSync(f).isFile(); @@ -106,7 +113,9 @@ function setup(fileContents: DirectoryJSON, options: { subdir?: string, initialS writeFile() { throw new Error('not implemented'); }, - realpath: identity, + realpath(f) { + return root + (vol.realpathSync(f, {encoding: 'utf8'})).substr(1); + }, getDirectories(d) { return mapDefined(vol.readdirSync(d, {encoding: 'utf8'}), (f) => { return this.directoryExists!(d + '/' + f) ? f : undefined; @@ -585,4 +594,127 @@ test('merges multiple level of circular dependencies II', (t) => { t.is(programState.getUpToDateResult(cwd + 'e.ts', '1234'), undefined); }); +test('uses earliest circular dependency', (t) => { + const files = { + 'tsconfig.json': '{}', + 'a.ts': 'import "./b"; import "./a1";', + 'a1.ts': 'import "./a2";', + 'a2.ts': 'import "./a3";', + 'a3.ts': 'import "./a1"; import "./a"; import "./a2";', + 'b.ts': 'export {};', + }; + const state = generateOldState('1234', files); + + let {programState, cwd, vol, program, compilerHost} = setup(files, {initialState: state}); + + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'a1.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'a2.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'a3.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + + vol.appendFileSync(cwd + 'b.ts', 'foo;'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'b.ts'); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'a1.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'a2.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'a3.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); +}); + +test('detects when all files are still the same, but resolution changed', (t) => { + const files = { + 'tsconfig.json': JSON.stringify({exclude: ['deps'], compilerOptions: {moduleResolution: 'node'}}), + 'a.ts': 'import "x"; import "y";', + 'deps/x/index.ts': 'export let x;', + 'deps/y/index.ts': 'export let y;', + 'node_modules': null, + }; + const state = generateOldState('1234', files, {symlinks: [ + ['node_modules/x', 'deps/x'], + ['node_modules/y', 'deps/y'], + ]}); + + let {programState, cwd} = setup(files, {initialState: state, symlinks: [ + ['node_modules/x', 'deps/x'], + ['node_modules/y', 'deps/y'], + ]}); + + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'deps/x/index.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'deps/y/index.ts', '1234'), []); + ({programState, cwd} = setup(files, {initialState: state, symlinks: [ + ['node_modules/x', 'deps/y'], + ['node_modules/y', 'deps/x'], + ]})); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.deepEqual(programState.getUpToDateResult(cwd + 'deps/x/index.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'deps/y/index.ts', '1234'), []); +}); + +test('detects added module augmentation', (t) => { + const files = { + 'tsconfig.json': '{}', + 'a.ts': 'export {};', + 'b.ts': 'export {};', + 'c.ts': 'import "./a";', + }; + const state = generateOldState('1234', files); + + let {programState, cwd, vol, program, compilerHost} = setup(files, {initialState: state}); + + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + + vol.appendFileSync(cwd + 'b.ts', 'declare module "./a" {};'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'b.ts'); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); +}); + +test('changes in unresolved dependencies', (t) => { + const files = { + 'tsconfig.json': '{}', + 'a.ts': 'import "x";', + 'b.ts': 'import "y";', + 'c.ts': 'declare module "x";', + }; + const state = generateOldState('1234', files); + + let {programState, cwd, vol, program, compilerHost} = setup(files, {initialState: state}); + + t.deepEqual(programState.getUpToDateResult(cwd + 'a.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'b.ts', '1234'), []); + t.deepEqual(programState.getUpToDateResult(cwd + 'c.ts', '1234'), []); + + vol.writeFileSync(cwd + 'c.ts', 'declare module "y";'); + program = ts.createProgram({ + oldProgram: program, + options: program.getCompilerOptions(), + rootNames: program.getRootFileNames(), + host: compilerHost, + }); + programState.update(program, cwd + 'c.ts'); + + t.is(programState.getUpToDateResult(cwd + 'a.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'b.ts', '1234'), undefined); + t.is(programState.getUpToDateResult(cwd + 'c.ts', '1234'), undefined); +}); From 37a13d9d95e9a0fc2cde945a4e612f618c90cffe Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 11 Feb 2021 21:46:40 +0100 Subject: [PATCH 33/37] test cache integration in runner --- .../packages/wotan/test/runner.spec.ts.md | 165 +++++++++++++++++- .../packages/wotan/test/runner.spec.ts.snap | Bin 558 -> 1605 bytes .../wotan/test/fixtures/cache/.wotanrc.yaml | 4 + packages/wotan/test/fixtures/cache/a.ts | 1 + packages/wotan/test/fixtures/cache/b.ts | 2 + .../fixtures/cache/tsconfig.fimbullintercache | 53 ++++++ .../wotan/test/fixtures/cache/tsconfig.json | 6 + packages/wotan/test/runner.spec.ts | 119 ++++++++++++- 8 files changed, 347 insertions(+), 3 deletions(-) create mode 100644 packages/wotan/test/fixtures/cache/.wotanrc.yaml create mode 100644 packages/wotan/test/fixtures/cache/a.ts create mode 100644 packages/wotan/test/fixtures/cache/b.ts create mode 100644 packages/wotan/test/fixtures/cache/tsconfig.fimbullintercache create mode 100644 packages/wotan/test/fixtures/cache/tsconfig.json diff --git a/baselines/packages/wotan/test/runner.spec.ts.md b/baselines/packages/wotan/test/runner.spec.ts.md index 55132bf05..0f0a4cd35 100644 --- a/baselines/packages/wotan/test/runner.spec.ts.md +++ b/baselines/packages/wotan/test/runner.spec.ts.md @@ -2,7 +2,7 @@ The actual snapshot is saved in `runner.spec.ts.snap`. -Generated by [AVA](https://ava.li). +Generated by [AVA](https://avajs.dev). ## multi-project @@ -39,3 +39,166 @@ Generated by [AVA](https://ava.li). }, ], ] + +## cache + + [ + [ + 'a.ts', + { + content: `console.log('foo');␊ + `, + findings: [], + fixes: 0, + }, + ], + [ + 'b.ts', + { + content: `debugger;␊ + // wotan-disable-next-line␊ + `, + findings: [ + { + end: { + character: 9, + line: 0, + position: 9, + }, + fix: { + replacements: [ + { + end: 9, + start: 0, + text: '', + }, + ], + }, + message: '\'debugger\' statements are forbidden.', + ruleName: 'no-debugger', + severity: 'error', + start: { + character: 0, + line: 0, + position: 0, + }, + }, + { + end: { + character: 26, + line: 1, + position: 36, + }, + fix: { + replacements: [ + { + end: 36, + start: 10, + text: '', + }, + ], + }, + message: 'Disable switch has no effect. All specified rules have no failures to disable.', + ruleName: 'useless-line-switch', + severity: 'error', + start: { + character: 0, + line: 1, + position: 10, + }, + }, + ], + fixes: 0, + }, + ], + ] + +## cache-outdated + + [ + [ + 'a.ts', + { + content: `console.log('foo');␊ + `, + findings: [], + fixes: 0, + }, + ], + [ + 'b.ts', + { + content: `debugger;␊ + // wotan-disable-next-line␊ + `, + findings: [ + { + end: { + character: 9, + line: 0, + position: 9, + }, + fix: { + replacements: [ + { + end: 9, + start: 0, + text: '', + }, + ], + }, + message: '\'debugger\' statements are forbidden.', + ruleName: 'no-debugger', + severity: 'error', + start: { + character: 0, + line: 0, + position: 0, + }, + }, + ], + fixes: 0, + }, + ], + ] + +## updated-state + + `cs: false␊ + files:␊ + - hash: '5901983184'␊ + - config: '-10079848193'␊ + hash: '5860249'␊ + result: []␊ + global:␊ + - 0␊ + lookup:␊ + a.ts: 0␊ + b.ts: 1␊ + options: '-10679607802'␊ + v: 1␊ + ` + +## cache-fix + + [ + [ + 'a.ts', + { + content: `console.log('foo');␊ + `, + findings: [], + fixes: 0, + }, + ], + [ + 'b.ts', + { + content: `␊ + ␊ + `, + findings: [], + fixes: 2, + }, + ], + ] diff --git a/baselines/packages/wotan/test/runner.spec.ts.snap b/baselines/packages/wotan/test/runner.spec.ts.snap index 4f4d644dc542b9356d5b66eac2f4e28b145b8a24..9552f7372385f5a4922de7f5fb00c79ad3d5ff8a 100644 GIT binary patch literal 1605 zcmV-L2DZ1vOs!!=)>3V6`a|fZG@b9{K{J^0t1IHyzs`y>ypQw6P(%6 zxrF?*ZykwzbL7CH&s*cG&c64A;LIMJM~H_69G^l^@Wk*x2s{LI3I0yOx7;w~t%Bf= z+S}WSAQ3?%!W=EfkkmDcX_k9E7uTk)vZ$(O7KPKg9$vf@p|=S_9L8N%Fs*1QMa!6+ zHD6lU!c6DNOJ?M6L5d~vnG7?QNVK)O|YFe_z#*rP9 z$_Arp-AE{@6w{)dji8ZN*_b?DGBQWgBd!^qYqAMuDAv|eikM;Oh7;-|Wuf}u+_1_P zx7}A`amSlm+}O%ODN%D$v65Mul})PYl%>@%xirR}ZL78ASCoL;# z-oS*V(_%q3+=i~a$za2DiZ)X4*~E5Stg#(MpapY?=Se)-oGLAvr~IPmqxNf|adz4%ef_2nhWgsgv)|g430k1)cs} zx>kcl)!;pd(P7{e@LiCrvzDr|yb8)S;C9HFz7R*htzlW(-D;YGEWFm!Y@mhjUGbuJ zC6pQNu4CBVf$1%m)Rd2OipNOO96_&BO(vxk^sW(!(umzBvm-S8z;LX0INskI9|$|s z_;#n23~oevW3i#(_&~gOxZl1-OHafHV|@d|g*kN3ylRcmr=FHFs-BS5g7KK7>iVX9 z&c3o`(gFGiFBtv_3_#S9H3J7DOvuLc4kc`ZfdG#Wfb zE4L6LEwL@L@a{)9n!2{ZHmX>7zsL0nTOnl`l}(cw_M1x;4HGi<1MP%PW);XTdn#{e zG$}(M@%#i+^+{W0B^WHG5QMB*62dF)-80vby1Kf*j?}NbCzaj1bq2k42Az#FsJImt zw>LU}7QR%w#bN~pKE>!9a0$2y#L-Ptz@7$&&eGxPu&2i1>TA#c=H&Rdr{+<~Zu9p8 zqd>O7*9CjO3zWS)t@@~R6qI*?Z-HNVdh1O)rL_2N+9?Hl*9-4MzIx)lRJY50u4c<& zGy%K@9P`!J;j<*HZ#D86H1bQpzdZd$Hu8D1(#S=rV7;$N>yG@tF3Ip8h<9$&BNqSw DMvf37 literal 558 zcmV+}0@3|JRzV-J>00000000B6 zQ@?8zK@@&(Z|_IeA9td)JOa9k++ATnFjyGu1k+eq=dL!k zXA$z}VF*xzrsvqgaD~^aj2nz~#%XMv!uoY%k_U(dYq#5g5nv7Q7~-A1WM)tD6+L~E z(;zQd`^5A!<7cH@P%6hlPK7388Oy?YRoG?0WybA_uvIn429C2LfHS#Qt`k|4_A{+} z&U%~oaL@5kv`-JwcAT)zi|tILMaKI@2G8{y(;tkL0N_@zu-#Z6Zi>`lZJ%~J9WUQv z$u{FH<6~9cI3WMZlJ7Zc083#-zEqZ<@aNXf?HBHwsa8%XSL~O??o1m;6XA%ziLTLw znAeytQ9Mjcd~ua$a1!x6w<>a4nN%aGHf*JKJ(Ih_`dygM;y<=u`^)xll)s{}8W6QNK@aEG-FT=SEl>{0{Ts4~RosV&*6@dag#4WKI~GNafff z?W>*T-kIF!ZCb4cc~R-eh^fBMCEZw1Vy!SvYDH4qzqidBeTUfr8rC9TjSoBBi7i_&fXPN{6040zXR{#J2 diff --git a/packages/wotan/test/fixtures/cache/.wotanrc.yaml b/packages/wotan/test/fixtures/cache/.wotanrc.yaml new file mode 100644 index 000000000..11bd2ecb7 --- /dev/null +++ b/packages/wotan/test/fixtures/cache/.wotanrc.yaml @@ -0,0 +1,4 @@ +rules: + no-debugger: error +settings: + foo: on diff --git a/packages/wotan/test/fixtures/cache/a.ts b/packages/wotan/test/fixtures/cache/a.ts new file mode 100644 index 000000000..81afa3157 --- /dev/null +++ b/packages/wotan/test/fixtures/cache/a.ts @@ -0,0 +1 @@ +console.log('foo'); diff --git a/packages/wotan/test/fixtures/cache/b.ts b/packages/wotan/test/fixtures/cache/b.ts new file mode 100644 index 000000000..30f41bed4 --- /dev/null +++ b/packages/wotan/test/fixtures/cache/b.ts @@ -0,0 +1,2 @@ +debugger; +// wotan-disable-next-line diff --git a/packages/wotan/test/fixtures/cache/tsconfig.fimbullintercache b/packages/wotan/test/fixtures/cache/tsconfig.fimbullintercache new file mode 100644 index 000000000..b14c4beb6 --- /dev/null +++ b/packages/wotan/test/fixtures/cache/tsconfig.fimbullintercache @@ -0,0 +1,53 @@ +state: + cs: false + files: + - config: '-10079848193' + hash: '5901983184' + result: [] + - config: '-10079848193' + hash: '-1055350110' + result: + - end: + character: 9 + line: 0 + position: 9 + fix: + replacements: + - end: 9 + start: 0 + text: '' + message: '''debugger'' statements are forbidden.' + ruleName: no-debugger + severity: error + start: + character: 0 + line: 0 + position: 0 + - end: + character: 26 + line: 1 + position: 36 + fix: + replacements: + - end: 36 + start: 10 + text: '' + message: >- + Disable switch has no effect. All specified rules have no failures + to disable. + ruleName: useless-line-switch + severity: error + start: + character: 0 + line: 1 + position: 10 + global: + - 1 + - 0 + lookup: + a.ts: 0 + b.ts: 1 + options: '-10679607802' + ts: 4.3.0-dev.20210210 + v: 1 +v: 1 diff --git a/packages/wotan/test/fixtures/cache/tsconfig.json b/packages/wotan/test/fixtures/cache/tsconfig.json new file mode 100644 index 000000000..db27c96f3 --- /dev/null +++ b/packages/wotan/test/fixtures/cache/tsconfig.json @@ -0,0 +1,6 @@ +{ + "compilerOptions": { + "noLib": true, + "types": [] + } +} diff --git a/packages/wotan/test/runner.spec.ts b/packages/wotan/test/runner.spec.ts index b06eec437..9b0b83e73 100644 --- a/packages/wotan/test/runner.spec.ts +++ b/packages/wotan/test/runner.spec.ts @@ -6,8 +6,10 @@ import { createDefaultModule } from '../src/di/default.module'; import { Runner } from '../src/runner'; import * as path from 'path'; import { NodeFileSystem } from '../src/services/default/file-system'; -import { FileSystem, MessageHandler, DirectoryService, FileSummary } from '@fimbul/ymir'; +import { FileSystem, MessageHandler, DirectoryService, FileSummary, StatePersistence } from '@fimbul/ymir'; import { unixifyPath } from '../src/utils'; +import { Linter } from '../src/linter'; +import * as yaml from 'js-yaml'; const directories: DirectoryService = { getCurrentDirectory() { return path.resolve('packages/wotan'); }, @@ -256,7 +258,7 @@ test('reports warnings while parsing tsconfig.json', (t) => { }); // TODO https://github.com/fimbullinter/wotan/issues/387 https://github.com/Microsoft/TypeScript/issues/26684 -test.skip('excludes symlinked typeRoots', (t) => { +test.failing('excludes symlinked typeRoots', (t) => { const container = new Container({defaultScope: BindingScopeEnum.Singleton}); container.bind(DirectoryService).toConstantValue(directories); interface FileMeta { @@ -462,3 +464,116 @@ test('supports linting multiple (overlapping) projects in one run', (t) => { ); t.snapshot(result, {id: 'multi-project'}); }); + +test('uses results from cache', (t) => { + const container = new Container({defaultScope: BindingScopeEnum.Singleton}); + container.bind(DirectoryService).toConstantValue(directories); + container.load(createCoreModule({}), createDefaultModule()); + container.get(Linter).getFindings = () => { throw new Error('should not be called'); }; + container.get(StatePersistence).saveState = () => {}; + const runner = container.get(Runner); + + const result = Array.from( + runner.lintCollection({ + cache: true, + config: undefined, + files: [], + exclude: [], + project: ['test/fixtures/cache'], + references: false, + fix: false, + extensions: undefined, + reportUselessDirectives: true, + }), + (entry): [string, FileSummary] => [unixifyPath(path.relative('packages/wotan/test/fixtures/cache', entry[0])), entry[1]], + ); + t.snapshot(result, {id: 'cache'}); +}); + +test('ignore cache if option is not enabled', (t) => { + const container = new Container({defaultScope: BindingScopeEnum.Singleton}); + container.bind(DirectoryService).toConstantValue(directories); + container.load(createCoreModule({}), createDefaultModule()); + container.get(StatePersistence).loadState = () => { throw new Error('should not be called'); }; + const runner = container.get(Runner); + + const result = Array.from( + runner.lintCollection({ + cache: false, + config: undefined, + files: [], + exclude: [], + project: ['test/fixtures/cache'], + references: false, + fix: false, + extensions: undefined, + reportUselessDirectives: true, + }), + (entry): [string, FileSummary] => [unixifyPath(path.relative('packages/wotan/test/fixtures/cache', entry[0])), entry[1]], + ); + t.snapshot(result, {id: 'cache'}); +}); + +test('discards cache if config changes', (t) => { + const container = new Container({defaultScope: BindingScopeEnum.Singleton}); + container.bind(DirectoryService).toConstantValue(directories); + container.load(createCoreModule({}), createDefaultModule()); + container.get(StatePersistence).saveState = () => { }; + const linter = container.get(Linter); + const getFindings = linter.getFindings; + const lintedFiles: string[] = []; + linter.getFindings = (...args) => { + lintedFiles.push(path.basename(args[0].fileName)); + return getFindings.apply(linter, args); + }; + const runner = container.get(Runner); + + const result = Array.from( + runner.lintCollection({ + cache: true, + config: undefined, + files: [], + exclude: [], + project: ['test/fixtures/cache'], + references: false, + fix: false, + extensions: undefined, + reportUselessDirectives: false, + }), + (entry): [string, FileSummary] => [unixifyPath(path.relative('packages/wotan/test/fixtures/cache', entry[0])), entry[1]], + ); + t.deepEqual(lintedFiles, ['a.ts', 'b.ts']); + t.snapshot(result, {id: 'cache-outdated'}); +}); + +test('cache and fix', (t) => { + const container = new Container({defaultScope: BindingScopeEnum.Singleton}); + container.bind(DirectoryService).toConstantValue(directories); + container.load(createCoreModule({}), createDefaultModule()); + container.get(StatePersistence).saveState = (_, {ts: _ts, ...rest}) => t.snapshot(yaml.dump(rest, {sortKeys: true}), {id: 'updated-state'}); + const linter = container.get(Linter); + const getFindings = linter.getFindings; + const lintedFiles: string[] = []; + linter.getFindings = (...args) => { + lintedFiles.push(path.basename(args[0].fileName)); + return getFindings.apply(linter, args); + }; + const runner = container.get(Runner); + + const result = Array.from( + runner.lintCollection({ + cache: true, + config: undefined, + files: [], + exclude: [], + project: ['test/fixtures/cache'], + references: false, + fix: true, + extensions: undefined, + reportUselessDirectives: true, + }), + (entry): [string, FileSummary] => [unixifyPath(path.relative('packages/wotan/test/fixtures/cache', entry[0])), entry[1]], + ); + t.deepEqual(lintedFiles, ['b.ts']); + t.snapshot(result, {id: 'cache-fix'}); +}); From 4dd2e45a670fc8577ea9bd6909939dcd2392cbb6 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 11 Feb 2021 21:51:55 +0100 Subject: [PATCH 34/37] omit case sensitivity from snapshot --- .../packages/wotan/test/runner.spec.ts.md | 3 +-- .../packages/wotan/test/runner.spec.ts.snap | Bin 1605 -> 1598 bytes packages/wotan/test/runner.spec.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/baselines/packages/wotan/test/runner.spec.ts.md b/baselines/packages/wotan/test/runner.spec.ts.md index 0f0a4cd35..a5802d6d0 100644 --- a/baselines/packages/wotan/test/runner.spec.ts.md +++ b/baselines/packages/wotan/test/runner.spec.ts.md @@ -164,8 +164,7 @@ Generated by [AVA](https://avajs.dev). ## updated-state - `cs: false␊ - files:␊ + `files:␊ - hash: '5901983184'␊ - config: '-10079848193'␊ hash: '5860249'␊ diff --git a/baselines/packages/wotan/test/runner.spec.ts.snap b/baselines/packages/wotan/test/runner.spec.ts.snap index 9552f7372385f5a4922de7f5fb00c79ad3d5ff8a..1111b3a2fa496feec9639353d93fbd83d556ba27 100644 GIT binary patch delta 1551 zcmV+q2JrdC489B^K~_N^Q*L2!b7*gLAa*he0suB23~r$J&A~4BaTgE)_xO<_CVwEv z1KC9b(Wp_Gwx{jD>`a-N?gECO;scO~@?b!SiG&AD1VSWHV<6FZA)qEIkeHargYf~> zcnR^sliy4`-S)KGZVek^LX$5&bLN}>|IhjV`R6;k2q8Ywer03Vr%OgxFBp8e`^2hq zFABu|jfDug^NUB5L&>%?3yHpF^Iaon&Irydkq9~X_J;@7ys;t@Ss=YN^ueol3(oAe zHbPE4cIC|#fq}ypo_p=1b;)Df1ZVcaxrF?Zxe(#jJvFVU|P{qik2}sYreFy zg_+Kkm(0lDf)q>SGZ|(qm3n&Uq;APtB&C>gLS+$+ZLuP%qOn2%>zfZCSdlg_pWp4U zxD$9utjuqqb}=0g-C{Z~y6FnEc@Pz_7`PvJ0XPm^@Nl^Z@~I2_1j_G#AEDj>>;_H) zKX|!E+p5y&cXs4|ns;hPJWNYDlZ$<2lh_VE0prXxeJlA9s%uuYYr4%v4&C!XDo>ND`oregituiX#Dz-d+{NC@AZAT1S=D&7e#I`+<`@y{lpK3MI3$Atx3+t;(TB#5iJJGsdBRLTFfX4-qXy3sNFKF zR(bpM4(tWgU1oRLGXI{fWj2gLa*E`xAR+ezD_idzu1AXz5c)Y%C*Pk1rzhjUB|G!1JheBsVN`n6pxU!f{ryJQ5vzkV|Ijw?;DQw4#)d@;{#!58eicVG)>D438-CKv- zVKLKk$oBWned6?#>iikQKY#%!n=xKSR;$i`O`vQBP6A&xP@SM(2IW^9P&*-B-GrJJ zsq@*EoNicDWfnw=PM6VwRHaL3I3XJ)$-21;ySut(WLtsDQ58!QN|tj&pOVx z)3l+F({MD}Q>=&ZQpwiRR#q`7(@aOiY?!)@E?7DA+d zCAMW2-u>uCQ`a`wMimS1_qaY`>!K{9vS~8Iesig!VM4}!pqnDPcS+MTmr5FadguZu&2R)QL}WUI_#-&r25M9zd0}d<*9iU);t^vy;-gGPP{_=l%o&qh9PRvNh|6|DC)Y2A_k|4B0Z3-@A5+!+@F005W- B_&ERo delta 1558 zcmV+x2I={}48;s0K~_N^Q*L2!b7*gLAa*he0stBYA`;~yZu4+8fxlY2jQx=!CVvo+ z2eOL>qEVwVZBN^Q*_kpk-31Lng$E!J^}&D;5(y8Q2!u$Y#z3O+LO@MaATcqK2jc^% z@e<;NC%>6?y6tJV-4Zs$geG5l=FB($|DW^!^Urs75kh>V{mRC!&z6j?UNHDd_lZ^K zb_&G)jfDug`^(3YL&>%?3yHpF^IfB7&Irz|BoT7posagfd2>Z1vOs!!=)>3V6`a|f zZG@b9{K{J^0t1IHyzs`y>ypQw6P(%6xrF?*ZykwzbL7CH&s*cG&c64A;LIMJM~H_6 z9G^l^@Wk*x2s{LnBmp9ShcG+>e8AZdD{O%H2?%F_A2_0~hO-thL^4LyQi_;i=!O&OBW0ob;M}mv7PsA3 zV{yluTin>nLMc&mQn8X*nw3qe>6E3@*vlwguBz0`v80lJR#=MKi!wo(U^XW$D{9`r zgr(DBK{nimuDr=$!*q%^Qt;Wtc3iBn9eBgNel%5J3GlF=D~%9eRbm6-9eZWbc-qo;qg_2p>kdqcO z3U9m+;s{=UI}xRBP??CFZYq|dYg3Q5I`1?RaXv7Wh!%t8R5@9AEoKx2?`h^i)NYwo ztGsi12lfK$F0;FAng7hzG8;xAIYn|$kdXU=m92LU*Q3P<2>l$Xlkd-h)0LM6o&H?9 zR)a;=;5~@ZVc-<-U68A@ma4M63d%L$cF38&5J$g%tzlW(-D;YGEWFm!Y@mhjUGbuJ zC6pQNu4CBVf$1%m)Rd2OipNOO96_&BO(vxk^sW(!(umzBvm-S8z;LX0INskI9|$|s z_;#n23~oevW3i#(_&~gOxZl1-OHafHV|@d|g*kN3ylRcmr=FHFs-BS5g7KK7>iVX9 z&c3pLWzq;{C`~w{UP;f{bz&Br3=Rzs#)jgtzOXc5-z-yt-U0lN04ssDjntsWDTpUa zYH&wUm0@_)9*T}0xQOBRz+a7oAs!_GV7^dK7=jo9p&wY!5rYjCq+}Y&9_TMuAwDZi zuOoTx-a6b4iO)O_Y@acIiMddVq z8azcSw-6#Nu`RRk?ngJ8y0*bKs#tix$Mp$YA!QkrO_Le+n@bf96EgM#?SxKd7051o zDsN~sDMKLf`~*|=Nn2$l7%Zj`gsfQ-!Yl6GGuM&2y1KuP)UUiJmEF5_2EBC#osBc7 zxD^(+H#&b7zEr!#Vg&|1#poPx3AhS>#L-Ptz@7$&&eGxPu&2i1>TA#c=H&Rdr{+<~ zZu9p8qd>O7*9CjO3zWS)t@@~R6qI*?Z-HNVdh1O)rL_2N+9?Hl*9-4MzIx)lRJY50 zu4c<&Gy%K@9P`!J;j<*HZ#D86H1bQpzdZd$Hu8D1(#S=rV7;$N>yG@tF3Io-ABcBu I(<2uE0Olk6MgRZ+ diff --git a/packages/wotan/test/runner.spec.ts b/packages/wotan/test/runner.spec.ts index 9b0b83e73..0f21dec0a 100644 --- a/packages/wotan/test/runner.spec.ts +++ b/packages/wotan/test/runner.spec.ts @@ -550,7 +550,7 @@ test('cache and fix', (t) => { const container = new Container({defaultScope: BindingScopeEnum.Singleton}); container.bind(DirectoryService).toConstantValue(directories); container.load(createCoreModule({}), createDefaultModule()); - container.get(StatePersistence).saveState = (_, {ts: _ts, ...rest}) => t.snapshot(yaml.dump(rest, {sortKeys: true}), {id: 'updated-state'}); + container.get(StatePersistence).saveState = (_, {ts: _ts, cs: _cs, ...rest}) => t.snapshot(yaml.dump(rest, {sortKeys: true}), {id: 'updated-state'}); const linter = container.get(Linter); const getFindings = linter.getFindings; const lintedFiles: string[] = []; From e67ac5d598e45cb8a6bb9993d6e0eb4b13a9dcbb Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 11 Feb 2021 22:01:07 +0100 Subject: [PATCH 35/37] make tests typescript version agnostic --- packages/wotan/test/runner.spec.ts | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/wotan/test/runner.spec.ts b/packages/wotan/test/runner.spec.ts index 0f21dec0a..6ce8b63f7 100644 --- a/packages/wotan/test/runner.spec.ts +++ b/packages/wotan/test/runner.spec.ts @@ -10,6 +10,9 @@ import { FileSystem, MessageHandler, DirectoryService, FileSummary, StatePersist import { unixifyPath } from '../src/utils'; import { Linter } from '../src/linter'; import * as yaml from 'js-yaml'; +import { DefaultStatePersistence } from '../src/services/default/state-persistence'; +import * as ts from 'typescript'; +import { CachedFileSystem } from '..'; const directories: DirectoryService = { getCurrentDirectory() { return path.resolve('packages/wotan'); }, @@ -465,12 +468,24 @@ test('supports linting multiple (overlapping) projects in one run', (t) => { t.snapshot(result, {id: 'multi-project'}); }); +@injectable() +class TsVersionAgnosticStatePersistence extends DefaultStatePersistence { + constructor(fs: CachedFileSystem) { + super(fs); + } + public loadState(project: string) { + const result = super.loadState(project); + return result && {...result, ts: ts.version}; + } + public saveState() {} +} + test('uses results from cache', (t) => { const container = new Container({defaultScope: BindingScopeEnum.Singleton}); container.bind(DirectoryService).toConstantValue(directories); + container.bind(StatePersistence).to(TsVersionAgnosticStatePersistence); container.load(createCoreModule({}), createDefaultModule()); container.get(Linter).getFindings = () => { throw new Error('should not be called'); }; - container.get(StatePersistence).saveState = () => {}; const runner = container.get(Runner); const result = Array.from( @@ -493,6 +508,7 @@ test('uses results from cache', (t) => { test('ignore cache if option is not enabled', (t) => { const container = new Container({defaultScope: BindingScopeEnum.Singleton}); container.bind(DirectoryService).toConstantValue(directories); + container.bind(StatePersistence).to(TsVersionAgnosticStatePersistence); container.load(createCoreModule({}), createDefaultModule()); container.get(StatePersistence).loadState = () => { throw new Error('should not be called'); }; const runner = container.get(Runner); @@ -517,8 +533,8 @@ test('ignore cache if option is not enabled', (t) => { test('discards cache if config changes', (t) => { const container = new Container({defaultScope: BindingScopeEnum.Singleton}); container.bind(DirectoryService).toConstantValue(directories); + container.bind(StatePersistence).to(TsVersionAgnosticStatePersistence); container.load(createCoreModule({}), createDefaultModule()); - container.get(StatePersistence).saveState = () => { }; const linter = container.get(Linter); const getFindings = linter.getFindings; const lintedFiles: string[] = []; @@ -549,6 +565,7 @@ test('discards cache if config changes', (t) => { test('cache and fix', (t) => { const container = new Container({defaultScope: BindingScopeEnum.Singleton}); container.bind(DirectoryService).toConstantValue(directories); + container.bind(StatePersistence).to(TsVersionAgnosticStatePersistence); container.load(createCoreModule({}), createDefaultModule()); container.get(StatePersistence).saveState = (_, {ts: _ts, cs: _cs, ...rest}) => t.snapshot(yaml.dump(rest, {sortKeys: true}), {id: 'updated-state'}); const linter = container.get(Linter); From 88d145218070c01bd57d08745ed367fd4cf12683 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 11 Feb 2021 22:06:31 +0100 Subject: [PATCH 36/37] fix import --- packages/wotan/test/runner.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/wotan/test/runner.spec.ts b/packages/wotan/test/runner.spec.ts index 6ce8b63f7..b14345c01 100644 --- a/packages/wotan/test/runner.spec.ts +++ b/packages/wotan/test/runner.spec.ts @@ -12,7 +12,7 @@ import { Linter } from '../src/linter'; import * as yaml from 'js-yaml'; import { DefaultStatePersistence } from '../src/services/default/state-persistence'; import * as ts from 'typescript'; -import { CachedFileSystem } from '..'; +import { CachedFileSystem } from '../src/services/cached-file-system'; const directories: DirectoryService = { getCurrentDirectory() { return path.resolve('packages/wotan'); }, From 31b26d6f950f8aa71d62afc4574266a762a43e69 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Fri, 12 Feb 2021 16:43:02 +0100 Subject: [PATCH 37/37] add docs --- packages/wotan/README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/wotan/README.md b/packages/wotan/README.md index cd74ef579..19765acc8 100644 --- a/packages/wotan/README.md +++ b/packages/wotan/README.md @@ -159,6 +159,7 @@ To detect unused or redundant comments you can use the `--report-useless-directi * `-p --project ` specifies the path to the `tsconfig.json` file to use. This option is used to find all files contained in your project. It also enables rules that require type information. This option can be used multiple times to specify multiple projects to lint. * `-r --references [true|false]` enables project references. Starting from the project specified with `-p --project` or the `tsconfig.json` in the current directory it will recursively follow all `"references"` and lint those projects. * `--report-useless-directives [true|false|error|warning|suggestion]` reports `// wotan-disable` and `// wotan-enable` comments that are redundant (i.e. rules are already disabled) or unused (there are no findings for the specified rules). Useless directives are reported as lint findings with the specified severity (`true` is converted to `error`). Those findings cannot be disabled by a disable comment. The findings are fixable which allow autofixing when used with the `--fix` option. +* `--cache` enables caching of lint results for projects. Can only be used with `-p --project` option. Read more about [caching](#caching). * `[...FILES]` specifies the files to lint. You can specify paths and glob patterns here. Note that all file paths are relative to the current working directory. Therefore `**/*.ts` doesn't match `../foo.ts`. @@ -233,6 +234,36 @@ Wotan respects these flags, too. That means it will not provide type information This ensures you won't get surprising lint findings caused by funky type inference in those files. You will still get reports for purely syntactic findings, i.e. rules that don't require type information. +### Caching + +Caching is done per project. Hence it requires type information. For every `tsconfig.json` it creates a `tsconfig.fimbullintercache` file that contains the state of the previous run. +The content of this file is not intended for use by other tools. All cache information is relative to the project directory. That means the cache can be checked into your VCS and reused by CI or your collegues. + +If a cache is available for a given project, Wotan can avoid linting files that are known to be unchanged. This can significantly reduce execution time. + +A file is treated as outdated and therefore linted if one of the following is true: + +* TypeScript version changed +* compilerOptions changed +* added, removed or outdated files that affect the global scope +* linter configuration for the file changed +* file has no cached lint result +* file content changed +* module resolution of imports or exports has changed + * dependency is added or removed + * ambient module or module augmentation is added, removed + * dependency is outdated + * if compilerOption `assumeChangesOnlyAffectDirectDependencies` is enabled, only checks direct dependencies + * otherwise recursively checks all transitive dependencies + +The following cases don't work well with caching: + +* rules accessing the file system or environment variables +* rules accessing other files in the project that are not global or dependencies of the current file +* linting the same project with different configurations -> only use caching for one of the configurations +* projects where all files are in the global scope +* updating the linter, rules or their (transitive) dependencies -> you need to manually remove the cache if you expect it to affect the lint result + ### Excluded Files If type information is available Wotan excludes all files you haven't written yourself. The following files are always excluded so you cannot explicitly include them: