Open5

microsoft/typescript のコードリーディング Day3 ~ LanguageService

mizchimizchi

https://zenn.dev/mizchi/scraps/3c30ea6fa9f8e5

https://zenn.dev/mizchi/scraps/80d47cbc601a5f

Day2 から色々とコードを読み進めていたが有意義な理解が進まなかった。が、WatcherProgram とか読んでてPR出したりしていた。

LanguageService から再開

LanguageService

LanguageServer 上でコードをこねくり回して、こういうコードを書いたのだが、どういうタイミングでコードがアップデートされるのかうまくに認識できなかった。

import ts from "typescript/lib/tsserverlibrary.js";
import fs from "node:fs";
import path from "node:path";
import { DocumentRegistry } from "typescript";

const tsconfig = ts.readConfigFile("./tsconfig.json", ts.sys.readFile);
const options = ts.parseJsonConfigFileContent(tsconfig.config, ts.sys, "./");
const defaultHost = ts.createCompilerHost(options.options);

const expandPath = (fname: string) => {
  if (fname.startsWith("/")) {
    return fname;
  }
  const root = process.cwd();
  return path.join(root, fname);
};

function applyRenameLocations(
  code: string,
  toName: string,
  renameLocations: readonly ts.RenameLocation[],
) {
  let current = code;
  let offset = 0;
  for (const loc of renameLocations) {
    const start = loc.textSpan.start;
    const end = loc.textSpan.start + loc.textSpan.length;
    current = current.slice(0, start + offset) + toName +
      current.slice(end + offset);
    offset += toName.length - (end - start);
  }
  return current;
}

type SnapshotManager = {
  readFileSnapshot(fileName: string): string | undefined;
  writeFileSnapshot(fileName: string, content: string): ts.SourceFile;
};

export interface InMemoryLanguageServiceHost extends ts.LanguageServiceHost {
  getSnapshotManager: (
    registory: DocumentRegistry,
  ) => SnapshotManager;
}

export function createInMemoryLanguageServiceHost(): InMemoryLanguageServiceHost {
  // read once, write on memory
  const fileContents = new Map<string, string>();
  const fileSnapshots = new Map<string, ts.IScriptSnapshot>();
  const fileVersions = new Map<string, number>();
  const fileDirtySet = new Set<string>();

  const getSnapshotManagerInternal: (
    registory: DocumentRegistry,
  ) => SnapshotManager = (registory: ts.DocumentRegistry) => {
    return {
      readFileSnapshot(fileName: string) {
        fileName = expandPath(fileName);
        console.log("[readFileSnapshot]", fileName);
        if (fileContents.has(fileName)) {
          return fileContents.get(fileName) as string;
        }
        return defaultHost.readFile(fileName);
      },
      writeFileSnapshot(fileName: string, content: string) {
        fileName = expandPath(fileName);
        const nextVersion = (fileVersions.get(fileName) || 0) + 1;
        // fileVersions.set(fileName, nextVersion);
        fileContents.set(fileName, content);
        console.log(
          "[writeFileSnapshot]",
          fileName,
          nextVersion,
          content.length,
        );
        fileDirtySet.add(fileName);
        const newSource = registory.updateDocument(
          fileName,
          serviceHost,
          ts.ScriptSnapshot.fromString(content),
          String(nextVersion),
        );
        return newSource;
      },
    };
  };

  const serviceHost: InMemoryLanguageServiceHost = {
    // getProjectVersion: () => {
    //   return projectVersion.toString();
    // },
    // hasInvalidatedResolution: () => false,
    getDefaultLibFileName: defaultHost.getDefaultLibFileName,
    fileExists: ts.sys.fileExists,
    readDirectory: ts.sys.readDirectory,
    directoryExists: ts.sys.directoryExists,
    getDirectories: ts.sys.getDirectories,
    getCurrentDirectory: defaultHost.getCurrentDirectory,
    getScriptFileNames: () => options.fileNames,
    getCompilationSettings: () => options.options,
    readFile: (fname, encode) => {
      fname = expandPath(fname);
      // console.log("[readFile]", fname);
      if (fileContents.has(fname)) {
        return fileContents.get(fname) as string;
      }
      const rawFileResult = ts.sys.readFile(fname, encode);
      if (rawFileResult) {
        fileContents.set(fname, rawFileResult);
        fileVersions.set(
          fname,
          (fileVersions.get(fname) || 0) + 1,
        );
      }
      return rawFileResult;
    },
    writeFile: (fileName, content) => {
      fileName = expandPath(fileName);
      console.log("[writeFile:mock]", fileName, content.length);
      // fileContents.set(fileName, content);
      // const version = fileVersions.get(fileName) || 0;
      // fileVersions.set(fileName, version + 1);
    },
    getScriptSnapshot: (fileName) => {
      fileName = expandPath(fileName);
      if (fileName.includes("src/index.ts")) {
        console.log("[getScriptSnapshot]", fileName);
      }
      if (fileSnapshots.has(fileName)) {
        return fileSnapshots.get(fileName)!;
      }
      const contentCache = fileContents.get(fileName);
      if (contentCache) {
        const newSnapshot = ts.ScriptSnapshot.fromString(contentCache);
        fileSnapshots.set(fileName, newSnapshot);
        return newSnapshot;
      }
      if (!fs.existsSync(fileName)) return;
      const raw = ts.sys.readFile(fileName, "utf8")!;
      const snopshot = ts.ScriptSnapshot.fromString(raw);
      fileSnapshots.set(fileName, snopshot);
      return snopshot;
    },
    getScriptVersion: (fileName) => {
      fileName = expandPath(fileName);
      const isDirty = fileDirtySet.has(fileName);
      if (isDirty) {
        const current = fileVersions.get(fileName) || 0;
        fileDirtySet.delete(fileName);
        fileVersions.set(fileName, current + 1);
      }
      return (fileVersions.get(fileName) || 0).toString();
    },
    getSnapshotManager: getSnapshotManagerInternal,
  };
  return serviceHost;
}

{
  // usage
  const prefs: ts.UserPreferences = {};
  const registory = ts.createDocumentRegistry();
  const serviceHost = createInMemoryLanguageServiceHost();
  const languageService = ts.createLanguageService(
    serviceHost,
    registory,
  );

  // languageService.
  const snapshotManager = serviceHost.getSnapshotManager(registory);

  // write src/index.ts and check types
  const raw = snapshotManager.readFileSnapshot("src/index.ts");
  const newSource = snapshotManager.writeFileSnapshot(
    "src/index.ts",
    raw + "\nconst y: number = x;",
  );

  // find scoped variables

  // languageService.getSemanticDiagnostics("src/index.ts");
  const program = languageService.getProgram()!;
  const checker = program.getTypeChecker();
  const localVariables = checker.getSymbolsInScope(
    newSource,
    ts.SymbolFlags.BlockScopedVariable,
  );

  // rename x to x_?
  const symbol = localVariables.find((s) => s.name === "x")!;
  const renameLocations = languageService.findRenameLocations(
    "src/index.ts",
    symbol.valueDeclaration!.getStart(),
    false,
    false,
    prefs,
  );
  const targets = new Set(renameLocations!.map((loc) => loc.fileName));

  let current = snapshotManager.readFileSnapshot("src/index.ts")!;
  for (const target of targets) {
    const renameLocationsToTarget = renameLocations!.filter(
      (loc) => expandPath(target) === expandPath(loc.fileName),
    );
    const newSymbolName = `${symbol.name}_${
      Math.random().toString(36).slice(2)
    }`;
    current = applyRenameLocations(
      current,
      newSymbolName,
      renameLocationsToTarget,
    );
  }
  snapshotManager.writeFileSnapshot("src/index.ts", current);
  const result = languageService.getSemanticDiagnostics("src/index.ts");
  console.log("post error", result.length);
  console.log(snapshotManager.readFileSnapshot("src/index.ts"));

  const oldProgram = program;
  {
    // rename y to y_?
    const program = languageService.getProgram()!;
    const program2 = languageService.getProgram()!;
    console.log(
      "------- program updated",
      program !== oldProgram,
      program2 === program,
    );
    const checker = program.getTypeChecker();
    const newSource = program.getSourceFile("src/index.ts")!;
    const localVariables = checker.getSymbolsInScope(
      newSource,
      ts.SymbolFlags.BlockScopedVariable,
    );
    const symbol = localVariables.find((s) => s.name === "y")!;
    const renameLocations = languageService.findRenameLocations(
      "src/index.ts",
      symbol.valueDeclaration!.getStart(),
      false,
      false,
      prefs,
    );
    const targets = new Set(renameLocations!.map((loc) => loc.fileName));
    let current = snapshotManager.readFileSnapshot("src/index.ts")!;
    for (const target of targets) {
      const renameLocationsToTarget = renameLocations!.filter(
        (loc) => expandPath(target) === expandPath(loc.fileName),
      );
      const newSymbolName = `${symbol.name}_${
        Math.random().toString(36).slice(2)
      }`;
      current = applyRenameLocations(
        current,
        newSymbolName,
        renameLocationsToTarget,
      );
    }
    snapshotManager.writeFileSnapshot("src/index.ts", current);
    const result = languageService.getSemanticDiagnostics("src/index.ts");
    console.log("post error", result.length);
    console.log(snapshotManager.readFileSnapshot("src/index.ts"));
  }
}

これはインメモリでファイルキャッシュを作って、実際のFSには書き込まず多段階でソースを変形したい、という考えで作った languageServer。

これは勉強した後に色々修正した結果、うまく動くのだが、最初は二回目の findRenameLocations がうまく動かず、結果としては一度書き換えた後に getProgram し直すことで正しくソースの位置が更新された。

ここに注目してほしい。

  const oldProgram = program;
  {
    // rename y to y_?
    const program = languageService.getProgram()!;
    const program2 = languageService.getProgram()!;
    console.log(
      "------- program updated",
      program !== oldProgram,
      program2 === program,
    );

これは true true となる。つまり sourceFile を触らない限りは、Program は更新されないが、少しでもコードを触ると program が再構築されている。

気になったこと

  • program のライフサイクルはどうなっているか
  • この getScriptSnapshotgetScriptVersion がどう使われているかを調べる
    • scriptVersion が更新されたら snapshot が更新されると理解している
  • program 参照がどう切り替わっているか調べる。
  • findRenameLocations でソースの位置が切り替わっていないのは、自分の実装が悪いのか、それとも明示的なキャッシュ破棄が必要なのか調べる。

挙動を見るに、 getScriptVersion でバージョンを上げても getScriptSnapshot がキャッシュを持っていると更新されない。それを確かめる。

mizchimizchi

SyntaxTreeCache

というわけで、getScriptSnapshot を使ってる場所を確認したら、 SyntaxTreeCache にたどり着いた。

class SyntaxTreeCache {
    // For our syntactic only features, we also keep a cache of the syntax tree for the
    // currently edited file.
    private currentFileName: string | undefined;
    private currentFileVersion: string | undefined;
    private currentFileScriptSnapshot: IScriptSnapshot | undefined;
    private currentSourceFile: SourceFile | undefined;

    constructor(private host: LanguageServiceHost) {
    }

    public getCurrentSourceFile(fileName: string): SourceFile {
        const scriptSnapshot = this.host.getScriptSnapshot(fileName);
        if (!scriptSnapshot) {
            // The host does not know about this file.
            throw new Error("Could not find file: '" + fileName + "'.");
        }

        const scriptKind = getScriptKind(fileName, this.host);
        const version = this.host.getScriptVersion(fileName);
        let sourceFile: SourceFile | undefined;

        if (this.currentFileName !== fileName) {
            // This is a new file, just parse it
            const options: CreateSourceFileOptions = {
                languageVersion: ScriptTarget.Latest,
                impliedNodeFormat: getImpliedNodeFormatForFile(
                    toPath(fileName, this.host.getCurrentDirectory(), this.host.getCompilerHost?.()?.getCanonicalFileName || hostGetCanonicalFileName(this.host)),
                    this.host.getCompilerHost?.()?.getModuleResolutionCache?.()?.getPackageJsonInfoCache(),
                    this.host,
                    this.host.getCompilationSettings()
                ),
                setExternalModuleIndicator: getSetExternalModuleIndicator(this.host.getCompilationSettings())
            };
            sourceFile = createLanguageServiceSourceFile(fileName, scriptSnapshot, options, version, /*setNodeParents*/ true, scriptKind);
        }
        else if (this.currentFileVersion !== version) {
            // This is the same file, just a newer version. Incrementally parse the file.
            const editRange = scriptSnapshot.getChangeRange(this.currentFileScriptSnapshot!);
            sourceFile = updateLanguageServiceSourceFile(this.currentSourceFile!, scriptSnapshot, version, editRange);
        }

        if (sourceFile) {
            // All done, ensure state is up to date
            this.currentFileVersion = version;
            this.currentFileName = fileName;
            this.currentFileScriptSnapshot = scriptSnapshot;
            this.currentSourceFile = sourceFile;
        }

        return this.currentSourceFile!;
    }
}
  • fileName に対して getScriptSnapshot() を叩いて、存在すればそれを返す
  • もし新旧の fileName が変わっていれば createLanguageServiceSourceFile で生成
  • もし getScriptVersion() の結果が変わっていれば、 updateLanguageServiceSourceFile(...) で更新。このとき editRange を渡している
  • 新しい sourceFile を返す
export function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange | undefined, aggressiveChecks?: boolean): SourceFile {
    // If we were given a text change range, and our version or open-ness changed, then
    // incrementally parse this file.
    if (textChangeRange) {
        if (version !== sourceFile.version) {
            let newText: string;

            // grab the fragment from the beginning of the original text to the beginning of the span
            const prefix = textChangeRange.span.start !== 0
                ? sourceFile.text.substr(0, textChangeRange.span.start)
                : "";

            // grab the fragment from the end of the span till the end of the original text
            const suffix = textSpanEnd(textChangeRange.span) !== sourceFile.text.length
                ? sourceFile.text.substr(textSpanEnd(textChangeRange.span))
                : "";

            if (textChangeRange.newLength === 0) {
                // edit was a deletion - just combine prefix and suffix
                newText = prefix && suffix ? prefix + suffix : prefix || suffix;
            }
            else {
                // it was actual edit, fetch the fragment of new text that correspond to new span
                const changedText = scriptSnapshot.getText(textChangeRange.span.start, textChangeRange.span.start + textChangeRange.newLength);
                // combine prefix, changed text and suffix
                newText = prefix && suffix
                    ? prefix + changedText + suffix
                    : prefix
                        ? (prefix + changedText)
                        : (changedText + suffix);
            }

            const newSourceFile = updateSourceFile(sourceFile, newText, textChangeRange, aggressiveChecks);
            setSourceFileFields(newSourceFile, scriptSnapshot, version);
            // after incremental parsing nameTable might not be up-to-date
            // drop it so it can be lazily recreated later
            newSourceFile.nameTable = undefined;

            // dispose all resources held by old script snapshot
            if (sourceFile !== newSourceFile && sourceFile.scriptSnapshot) {
                if (sourceFile.scriptSnapshot.dispose) {
                    sourceFile.scriptSnapshot.dispose();
                }

                sourceFile.scriptSnapshot = undefined;
            }

            return newSourceFile;
        }
    }

    const options: CreateSourceFileOptions = {
        languageVersion: sourceFile.languageVersion,
        impliedNodeFormat: sourceFile.impliedNodeFormat,
        setExternalModuleIndicator: sourceFile.setExternalModuleIndicator,
    };
    // Otherwise, just create a new source file.
    return createLanguageServiceSourceFile(sourceFile.fileName, scriptSnapshot, options, version, /*setNodeParents*/ true, sourceFile.scriptKind);
}
  • もし snapshot に textChangeRange がある場合、その区間を snapshot から取り出して、それ以外は sourceFile から取り出して新しいコードに合成する
  • updateSourceFile(sourceFile, newText, textChangeRange, aggressiveChecks) で textRange だけ sourceFile を更新する?
  • setSourceFileFields で soureFile に snapshot, version を書き込む
  • 古い snapshot を dispose する。
  • 最後に createLanguageServiceSourceFile で返す。ドキュメントレジストリなどに書き込んでる?

Parser's updateSourceFile() with textChangeRange

AST の部分更新をすると予想して読む。

export function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks = false): SourceFile {
    const newSourceFile = IncrementalParser.updateSourceFile(sourceFile, newText, textChangeRange, aggressiveChecks);
    // Because new source file node is created, it may not have the flag PossiblyContainDynamicImport. This is the case if there is no new edit to add dynamic import.
    // We will manually port the flag to the new source file.
    (newSourceFile as Mutable<SourceFile>).flags |= (sourceFile.flags & NodeFlags.PermanentlySetIncrementalFlags);
    return newSourceFile;
}

IncrementalParser.updateSourceFile(sourceFile, newText, textChangeRange, aggressiveChecks); を呼んでる。

namespace IncrementalParser {
    export function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks: boolean): SourceFile {
        aggressiveChecks = aggressiveChecks || Debug.shouldAssert(AssertionLevel.Aggressive);

        checkChangeRange(sourceFile, newText, textChangeRange, aggressiveChecks);
        if (textChangeRangeIsUnchanged(textChangeRange)) {
            // if the text didn't change, then we can just return our current source file as-is.
            return sourceFile;
        }

変化がなければそのままかえす。区間をチェックする関数

    function checkChangeRange(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks: boolean) {
        const oldText = sourceFile.text;
        if (textChangeRange) {
            Debug.assert((oldText.length - textChangeRange.span.length + textChangeRange.newLength) === newText.length);

            if (aggressiveChecks || Debug.shouldAssert(AssertionLevel.VeryAggressive)) {
                const oldTextPrefix = oldText.substr(0, textChangeRange.span.start);
                const newTextPrefix = newText.substr(0, textChangeRange.span.start);
                Debug.assert(oldTextPrefix === newTextPrefix);

                const oldTextSuffix = oldText.substring(textSpanEnd(textChangeRange.span), oldText.length);
                const newTextSuffix = newText.substring(textSpanEnd(textChangeRangeNewSpan(textChangeRange)), newText.length);
                Debug.assert(oldTextSuffix === newTextSuffix);
            }
        }
    }

文字列を更新するだけ

updateLanguageServiceSourceFile から

            const newSourceFile = updateSourceFile(sourceFile, newText, textChangeRange, aggressiveChecks);
            setSourceFileFields(newSourceFile, scriptSnapshot, version);

// ...
export function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks = false): SourceFile {
    const newSourceFile = IncrementalParser.updateSourceFile(sourceFile, newText, textChangeRange, aggressiveChecks);
    // Because new source file node is created, it may not have the flag PossiblyContainDynamicImport. This is the case if there is no new edit to add dynamic import.
    // We will manually port the flag to the new source file.
    (newSourceFile as Mutable<SourceFile>).flags |= (sourceFile.flags & NodeFlags.PermanentlySetIncrementalFlags);
    return newSourceFile;
}

IncrementalParser で部分的にパースして sourceFile を更新。
これはまだ静的解析がかかってないことに注意。

だいぶ戻る。 SyntaxTreeCache をどのように活用しているか。

createLanguageService がインスタンス化している。

    const syntaxTreeCache: SyntaxTreeCache = new SyntaxTreeCache(host);

vscode で findReferences してみたが、利用箇所が多すぎる。とりあえずあらゆる箇所で getCurrentSourceFile が呼ばれて、この cache から解決されている。

で、結局困ってるのが、Snapshot を多段で書き換えたときに languageService.findRenameLocations が古い情報を返すことで、これはどこで解決できる? findReferences の実装を見てみよう。

たぶんここ

    function findRenameLocations(fileName: string, position: number, findInStrings: boolean, findInComments: boolean, preferences?: UserPreferences | boolean): RenameLocation[] | undefined {
        synchronizeHostData();
        const sourceFile = getValidSourceFile(fileName);
        const node = getAdjustedRenameLocation(getTouchingPropertyName(sourceFile, position));
        if (!Rename.nodeIsEligibleForRename(node)) return undefined;
        if (isIdentifier(node) && (isJsxOpeningElement(node.parent) || isJsxClosingElement(node.parent)) && isIntrinsicJsxName(node.escapedText)) {
            const { openingElement, closingElement } = node.parent.parent;
            return [openingElement, closingElement].map((node): RenameLocation => {
                const textSpan = createTextSpanFromNode(node.tagName, sourceFile);
                return {
                    fileName: sourceFile.fileName,
                    textSpan,
                    ...FindAllReferences.toContextSpan(textSpan, sourceFile, node.parent)
                };
            });
        }
        else {
            const quotePreference = getQuotePreference(sourceFile, preferences ?? emptyOptions);
            const providePrefixAndSuffixTextForRename = typeof preferences === "boolean" ? preferences : preferences?.providePrefixAndSuffixTextForRename;
            return getReferencesWorker(node, position, { findInStrings, findInComments, providePrefixAndSuffixTextForRename, use: FindAllReferences.FindReferencesUse.Rename },
                (entry, originalNode, checker) => FindAllReferences.toRenameLocation(entry, originalNode, checker, providePrefixAndSuffixTextForRename || false, quotePreference));
        }
    }

synchronizeHostData してるはずだが、 getValidSorcueFile の結果が古いのか?

mizchimizchi

synchronizeHostData は watchProgram でも見たが、古いソースを sync してるはず...

    function getValidSourceFile(fileName: string): SourceFile {
        const sourceFile = program.getSourceFile(fileName);
        if (!sourceFile) {
            const error: Error & PossibleProgramFileInfo = new Error(`Could not find source file: '${fileName}'.`);

            // We've been having trouble debugging this, so attach sidecar data for the tsserver log.
            // See https://github.com/microsoft/TypeScript/issues/30180.
            error.ProgramFiles = program.getSourceFiles().map(f => f.fileName);

            throw error;
        }
        return sourceFile;
    }

たぶん解決してる program が古そう。

なんかこの挙動見覚えある気がしてきた。vscode で変更を多段で重ねたとき、最初の変更しか当たってないやつ。

PR 送るべきかは不明だが、一旦どこのキャッシュを捨てればこれが意図した挙動になるかを調べたい。 service.getProgram したときに更新されることがわかっている。

まず synchronizeHostData と getProgarm で起きる差分について考えてみよう。 getProgram から先に見る。

    // TODO: GH#18217 frequently asserted as defined
    function getProgram(): Program | undefined {
        if (languageServiceMode === LanguageServiceMode.Syntactic) {
            Debug.assert(program === undefined);
            return undefined;
        }

        synchronizeHostData();

        return program;
    }

あれ、一緒だ。逆に getSemanticDiagnostics をみる。

    /**
     * getSemanticDiagnostics return array of Diagnostics. If '-d' is not enabled, only report semantic errors
     * If '-d' enabled, report both semantic and emitter errors
     */
    function getSemanticDiagnostics(fileName: string): Diagnostic[] {
        synchronizeHostData();

        const targetSourceFile = getValidSourceFile(fileName);

        // Only perform the action per file regardless of '-out' flag as LanguageServiceHost is expected to call this function per file.
        // Therefore only get diagnostics for given file.

        const semanticDiagnostics = program.getSemanticDiagnostics(targetSourceFile, cancellationToken);
        if (!getEmitDeclarations(program.getCompilerOptions())) {
            return semanticDiagnostics.slice();
        }

        // If '-d' is enabled, check for emitter error. One example of emitter error is export class implements non-export interface
        const declarationDiagnostics = program.getDeclarationDiagnostics(targetSourceFile, cancellationToken);
        return [...semanticDiagnostics, ...declarationDiagnostics];
    }

やはり一緒に見える。ちゃんとsynchronize~を読む。

    function synchronizeHostData(): void {
        Debug.assert(languageServiceMode !== LanguageServiceMode.Syntactic);
        // perform fast check if host supports it
        if (host.getProjectVersion) {
            const hostProjectVersion = host.getProjectVersion();
            if (hostProjectVersion) {
                if (lastProjectVersion === hostProjectVersion && !host.hasChangedAutomaticTypeDirectiveNames?.()) {
                    return;
                }

                lastProjectVersion = hostProjectVersion;
            }
        }

        const typeRootsVersion = host.getTypeRootsVersion ? host.getTypeRootsVersion() : 0;
        if (lastTypesRootVersion !== typeRootsVersion) {
            log("TypeRoots version has changed; provide new program");
            program = undefined!; // TODO: GH#18217
            lastTypesRootVersion = typeRootsVersion;
        }

host.getProjectVersion という概念があって、これを自分で定義すれば program の明示的な入れ替えは可能か。でもオーバーヘッド大きそう。

プロジェクト自体の設定が変わってないかチェック。自分にはあまり関係ないか?

        const typeRootsVersion = host.getTypeRootsVersion ? host.getTypeRootsVersion() : 0;
        if (lastTypesRootVersion !== typeRootsVersion) {
            log("TypeRoots version has changed; provide new program");
            program = undefined!; // TODO: GH#18217
            lastTypesRootVersion = typeRootsVersion;
        }

        // This array is retained by the program and will be used to determine if the program is up to date,
        // so we need to make a copy in case the host mutates the underlying array - otherwise it would look
        // like every program always has the host's current list of root files.
        const rootFileNames = host.getScriptFileNames().slice();

        // Get a fresh cache of the host information
        const newSettings = host.getCompilationSettings() || getDefaultCompilerOptions();
        const hasInvalidatedResolutions: HasInvalidatedResolutions = host.hasInvalidatedResolutions || returnFalse;
        const hasInvalidatedLibResolutions = maybeBind(host, host.hasInvalidatedLibResolutions) || returnFalse;
        const hasChangedAutomaticTypeDirectiveNames = maybeBind(host, host.hasChangedAutomaticTypeDirectiveNames);
        const projectReferences = host.getProjectReferences?.();
        let parsedCommandLines: Map<Path, ParsedCommandLine | false> | undefined;

いや、ここは関係ありそう。

        const hasInvalidatedResolutions: HasInvalidatedResolutions = host.hasInvalidatedResolutions || returnFalse;

//...
/** @internal */
export type HasInvalidatedResolutions = (sourceFile: Path) => boolean;
/** @internal */
export type HasInvalidatedLibResolutions = (libFileName: string) => boolean;
/** @internal */
export type HasChangedAutomaticTypeDirectiveNames = () => boolean;

compilerHost を作り直して...

        // Now create a new compiler
        let compilerHost: CompilerHost | undefined = {
            getSourceFile: getOrCreateSourceFile,
            getSourceFileByPath: getOrCreateSourceFileByPath,
            getCancellationToken: () => cancellationToken,
            getCanonicalFileName,
            useCaseSensitiveFileNames: () => useCaseSensitiveFileNames,
            getNewLine: () => getNewLineCharacter(newSettings),
            getDefaultLibFileName: options => host.getDefaultLibFileName(options),
            writeFile: noop,
            getCurrentDirectory: () => currentDirectory,
            fileExists: fileName => host.fileExists(fileName),
            readFile: fileName => host.readFile && host.readFile(fileName),
            getSymlinkCache: maybeBind(host, host.getSymlinkCache),
            realpath: maybeBind(host, host.realpath),
            directoryExists: directoryName => {
                return directoryProbablyExists(directoryName, host);
            },
            getDirectories: path => {
                return host.getDirectories ? host.getDirectories(path) : [];
            },
            readDirectory: (path: string, extensions?: readonly string[], exclude?: readonly string[], include?: readonly string[], depth?: number) => {
                Debug.checkDefined(host.readDirectory, "'LanguageServiceHost.readDirectory' must be implemented to correctly process 'projectReferences'");
                return host.readDirectory!(path, extensions, exclude, include, depth);
            },
            onReleaseOldSourceFile,
            onReleaseParsedCommandLine,
            hasInvalidatedResolutions,
            hasInvalidatedLibResolutions,
            hasChangedAutomaticTypeDirectiveNames,
            trace: maybeBind(host, host.trace),
            resolveModuleNames: maybeBind(host, host.resolveModuleNames),
            getModuleResolutionCache: maybeBind(host, host.getModuleResolutionCache),
            createHash: maybeBind(host, host.createHash),
            resolveTypeReferenceDirectives: maybeBind(host, host.resolveTypeReferenceDirectives),
            resolveModuleNameLiterals: maybeBind(host, host.resolveModuleNameLiterals),
            resolveTypeReferenceDirectiveReferences: maybeBind(host, host.resolveTypeReferenceDirectiveReferences),
            resolveLibrary: maybeBind(host, host.resolveLibrary),
            useSourceOfProjectReferenceRedirect: maybeBind(host, host.useSourceOfProjectReferenceRedirect),
            getParsedCommandLine,
        };

同一性のチェック。構築したプログラムが最新なら何もしない。

        // The call to isProgramUptoDate below may refer back to documentRegistryBucketKey;
        // calculate this early so it's not undefined if downleveled to a var (or, if emitted
        // as a const variable without downleveling, doesn't crash).
        const documentRegistryBucketKey = documentRegistry.getKeyForCompilationSettings(newSettings);
        let releasedScriptKinds: Set<Path> | undefined = new Set();

        // If the program is already up-to-date, we can reuse it
        if (isProgramUptoDate(program, rootFileNames, newSettings, (_path, fileName) => host.getScriptVersion(fileName), fileName => compilerHost!.fileExists(fileName), hasInvalidatedResolutions, hasInvalidatedLibResolutions, hasChangedAutomaticTypeDirectiveNames, getParsedCommandLine, projectReferences)) {
            compilerHost = undefined;
            parsedCommandLines = undefined;
            releasedScriptKinds = undefined;
            return;
        }

あー、可能性を一つ見つけた。自分が作った InMemoryLanguageService だと、内部的に write した時点で getScriptVersion を上げてしまってるので、file.version が一致していて、 ここの同一性確認で引っかからず更新されてない可能性がある。で、どこかの getCurrentSourceFile で触られた時点で更新される。

あるいは明示的に hasInvalidatedResolutions を実装することで解決できそう。

一応、isProgramUptoDate の実装を見てみよう。

/**
 * Determines if program structure is upto date or needs to be recreated
 *
 * @internal
 */
export function isProgramUptoDate(
    program: Program | undefined,
    rootFileNames: string[],
    newOptions: CompilerOptions,
    getSourceVersion: (path: Path, fileName: string) => string | undefined,
    fileExists: (fileName: string) => boolean,
    hasInvalidatedResolutions: HasInvalidatedResolutions,
    hasInvalidatedLibResolutions: HasInvalidatedLibResolutions,
    hasChangedAutomaticTypeDirectiveNames: HasChangedAutomaticTypeDirectiveNames | undefined,
    getParsedCommandLine: (fileName: string) => ParsedCommandLine | undefined,
    projectReferences: readonly ProjectReference[] | undefined
): boolean {
    // If we haven't created a program yet or have changed automatic type directives, then it is not up-to-date
    if (!program || hasChangedAutomaticTypeDirectiveNames?.()) return false;

    // If root file names don't match
    if (!arrayIsEqualTo(program.getRootFileNames(), rootFileNames)) return false;

    let seenResolvedRefs: ResolvedProjectReference[] | undefined;

    // If project references don't match
    if (!arrayIsEqualTo(program.getProjectReferences(), projectReferences, projectReferenceUptoDate)) return false;

    // If any file is not up-to-date, then the whole program is not up-to-date
    if (program.getSourceFiles().some(sourceFileNotUptoDate)) return false;

    // If any of the missing file paths are now created
    if (program.getMissingFilePaths().some(fileExists)) return false;

    const currentOptions = program.getCompilerOptions();
    // If the compilation settings do no match, then the program is not up-to-date
    if (!compareDataObjects(currentOptions, newOptions)) return false;

    // If library resolution is invalidated, then the program is not up-to-date
    if (program.resolvedLibReferences && forEachEntry(program.resolvedLibReferences, (_value, libFileName) => hasInvalidatedLibResolutions(libFileName))) return false;

    // If everything matches but the text of config file is changed,
    // error locations can change for program options, so update the program
    if (currentOptions.configFile && newOptions.configFile) return currentOptions.configFile.text === newOptions.configFile.text;

    return true;

    function sourceFileNotUptoDate(sourceFile: SourceFile) {
        return !sourceFileVersionUptoDate(sourceFile) ||
            hasInvalidatedResolutions(sourceFile.path);
    }

    function sourceFileVersionUptoDate(sourceFile: SourceFile) {
        return sourceFile.version === getSourceVersion(sourceFile.resolvedPath, sourceFile.fileName);
    }

    function projectReferenceUptoDate(oldRef: ProjectReference, newRef: ProjectReference, index: number) {
        return projectReferenceIsEqualTo(oldRef, newRef) &&
            resolvedProjectReferenceUptoDate(program!.getResolvedProjectReferences()![index], oldRef);
    }

    function resolvedProjectReferenceUptoDate(oldResolvedRef: ResolvedProjectReference | undefined, oldRef: ProjectReference): boolean {
        if (oldResolvedRef) {
                // Assume true
            if (contains(seenResolvedRefs, oldResolvedRef)) return true;

            const refPath = resolveProjectReferencePath(oldRef);
            const newParsedCommandLine = getParsedCommandLine(refPath);

            // Check if config file exists
            if (!newParsedCommandLine) return false;

            // If change in source file
            if (oldResolvedRef.commandLine.options.configFile !== newParsedCommandLine.options.configFile) return false;

            // check file names
            if (!arrayIsEqualTo(oldResolvedRef.commandLine.fileNames, newParsedCommandLine.fileNames)) return false;

            // Add to seen before checking the referenced paths of this config file
            (seenResolvedRefs || (seenResolvedRefs = [])).push(oldResolvedRef);

            // If child project references are upto date, this project reference is uptodate
            return !forEach(oldResolvedRef.references, (childResolvedRef, index) =>
                !resolvedProjectReferenceUptoDate(childResolvedRef, oldResolvedRef.commandLine.projectReferences![index]));
        }

        // In old program, not able to resolve project reference path,
        // so if config file doesnt exist, it is uptodate.
        const refPath = resolveProjectReferencePath(oldRef);
        return !getParsedCommandLine(refPath);
    }
}

ファイル更新時に引っかかるべきはここかな。

    // If any file is not up-to-date, then the whole program is not up-to-date
    if (program.getSourceFiles().some(sourceFileNotUptoDate)) return false;
    function sourceFileNotUptoDate(sourceFile: SourceFile) {
        return !sourceFileVersionUptoDate(sourceFile) ||
            hasInvalidatedResolutions(sourceFile.path);
    }

    function sourceFileVersionUptoDate(sourceFile: SourceFile) {
        return sourceFile.version === getSourceVersion(sourceFile.resolvedPath, sourceFile.fileName);
    }

hostSynchronize~ の次に進む

        const options: CreateProgramOptions = {
            rootNames: rootFileNames,
            options: newSettings,
            host: compilerHost,
            oldProgram: program,
            projectReferences
        };
        program = createProgram(options);

        // 'getOrCreateSourceFile' depends on caching but should be used past this point.
        // After this point, the cache needs to be cleared to allow all collected snapshots to be released
        compilerHost = undefined;
        parsedCommandLines = undefined;
        releasedScriptKinds = undefined;

        // We reset this cache on structure invalidation so we don't hold on to outdated files for long; however we can't use the `compilerHost` above,
        // Because it only functions until `hostCache` is cleared, while we'll potentially need the functionality to lazily read sourcemap files during
        // the course of whatever called `synchronizeHostData`
        sourceMapper.clearCache();

        // Make sure all the nodes in the program are both bound, and have their parent
        // pointers set property.
        program.getTypeChecker();
        return;

program が再生成されて、新しい TypeChecker で型情報が更新される。
ついでに sourceMap の cache も捨てている。

mizchimizchi

プログラムのライフサイクル

わかったこととして、 program は基本的に解析ステップの状態ごとに作り直されるものであって、下手に再利用を考えないほうが良い

ScriptSnapshot の change range の設定方法

どういう風に設定するか

テストコードだと雑に注入されてる

src/testRunner/unittests/helpers.ts
        // Always treat any change as a full change.
        snapshot.getChangeRange = () => ts.createTextChangeRange(ts.createTextSpan(0, contents.length), contents.length);

tssserver だと行単位キャッシュみたいなのが実装されてる?

class LineIndexSnapshot implements IScriptSnapshot {
    constructor(readonly version: number, readonly cache: ScriptVersionCache, readonly index: LineIndex, readonly changesSincePreviousVersion: readonly TextChange[] = emptyArray) {
    }

    getText(rangeStart: number, rangeEnd: number) {
        return this.index.getText(rangeStart, rangeEnd - rangeStart);
    }

    getLength() {
        return this.index.getLength();
    }

    getChangeRange(oldSnapshot: IScriptSnapshot): TextChangeRange | undefined {
        if (oldSnapshot instanceof LineIndexSnapshot && this.cache === oldSnapshot.cache) {
            if (this.version <= oldSnapshot.version) {
                return unchangedTextChangeRange;
            }
            else {
                return this.cache.getTextChangesBetweenVersions(oldSnapshot.version, this.version);
            }
        }
    }
}
    getTextChangesBetweenVersions(oldVersion: number, newVersion: number) {
        if (oldVersion < newVersion) {
            if (oldVersion >= this.minVersion) {
                const textChangeRanges: TextChangeRange[] = [];
                for (let i = oldVersion + 1; i <= newVersion; i++) {
                    const snap = this.versions[this.versionToIndex(i)!]; // TODO: GH#18217
                    for (const textChange of snap.changesSincePreviousVersion) {
                        textChangeRanges.push(textChange.getTextChangeRange());
                    }
                }
                return collapseTextChangeRangesAcrossMultipleVersions(textChangeRanges);
            }
            else {
                return undefined;
            }
        }
        else {
            return unchangedTextChangeRange;
        }
    }

tsserver の内部で賢い ScriptVersionCache が使われているが、これを外部から触る方法はなさそう。

/** @internal */
export class ScriptVersionCache {
    private static readonly changeNumberThreshold = 8;
    private static readonly changeLengthThreshold = 256;
    private static readonly maxVersions = 8;

というわけで、自分で勝手に編集単位の snapshot 作っちゃって良さそう。例えばこう

function createSnapshotWithChange(content: string, changes: ts.TextChangeRange): ts.IScriptSnapshot {
  const snapshot: ts.IScriptSnapshot = {
    getText: (start, end) => content.slice(start, end),
    getLength: () => content.length,
    getChangeRange: () => changes,
  };
  return snapshot;
}
mizchimizchi

まとめ

  • LanguageService はソースコードの変更に応じて内部の program を再生成する
  • Program や TypeChecker はソースコードを変更するごとに作り直す必要がある
    • しかし前に読んだように createProgram は oldProgrma を引数にとれて、変更不要な SourceFile を引き継ぐ
  • コードの変更は documentRegistry.update() への変更だけでは不十分で、 host.getScriptVersion(fileName) で都度のバージョンを返す実装をして、 host.getScriptSnapshot(fileName) で実際に手作りした IScriptSnapshot を返すことができる
  • IScriptSnapshot の getChangeRange を定義することで変更レンジを返すことができ、 IncrementalParser はこれを元に部分的にASTを作り直す
  • 賢い snapshot の更新は tsserver の languageService host 内で実装されているが、それを自分から使う方法はできない。が、操作範囲が限定的なら手作りはそこまで難しくなさそう