Merge branch 'main' into aeisenberg/graph-viewer

Author: aeisenberg

.github/workflows/main.yml

@@ -135,7 +135,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'v2.7.6', 'v2.8.0', 'nightly']
+        version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'v2.7.6', 'v2.8.1', 'nightly']
     env:
       CLI_VERSION: ${{ matrix.version }}
       NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}

extensions/ql-vscode/CHANGELOG.md

@@ -2,10 +2,15 @@
 
 ## [UNRELEASED]
 
+- Fix a bug where database upgrades could not be resolved if some of the target pack's dependencies are outside of the workspace. [#1138](https://github.com/github/vscode-codeql/pull/1138)
+- Open the query server logs for query errors (instead of the extension log). This will make it easier to track down query errors. [#1158](https://github.com/github/vscode-codeql/pull/1158)
+- Fix a bug where queries took a long time to run if there are no folders in the workspace. [#1157](https://github.com/github/vscode-codeql/pull/1157)
+
 ## 1.5.11 - 10 February 2022
 
 - Fix a bug where invoking _View AST_ from the file explorer would not view the selected file. Instead it would view the active editor. Also, prevent the _View AST_ from appearing if the current selection includes a directory or multiple files. [#1113](https://github.com/github/vscode-codeql/pull/1113)
 - Add query history items as soon as a query is run, including new icons for each history item. [#1094](https://github.com/github/vscode-codeql/pull/1094)
+- Save query history items across restarts. Items will be saved for 30 days and can be overwritten by setting the `codeQL.queryHistory.ttl` configuration setting. [#1130](https://github.com/github/vscode-codeql/pull/1130)
 - Allow in-progress query items to be cancelled from the query history view. [#1105](https://github.com/github/vscode-codeql/pull/1105)
 
 ## 1.5.10 - 25 January 2022
@@ -32,7 +37,7 @@
 - Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
 - Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`) in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)
 - Allow case-insensitive project slugs for GitHub repositories when adding a CodeQL database from LGTM. [#978](https://github.com/github/vscode-codeql/pull/961)
-- Add a _CodeQL: Preview Query Help_ command to generate Markdown previews of `.qhelp` query help files. This command should only be run in trusted workspaces. See https://codeql.github.com/docs/codeql-cli/testing-query-help-files for more information about query help. [#988](https://github.com/github/vscode-codeql/pull/988)
+- Add a _CodeQL: Preview Query Help_ command to generate Markdown previews of `.qhelp` query help files. This command should only be run in trusted workspaces. See [the CodeQL CLI docs](https://codeql.github.com/docs/codeql-cli/testing-query-help-files) for more information about query help. [#988](https://github.com/github/vscode-codeql/pull/988)
 - Make "Open Referenced File" command accessible from the active editor menu. [#989](https://github.com/github/vscode-codeql/pull/989)
 - Fix a bug where result set names in the result set drop-down were disappearing when viewing a sorted table. [#1007](https://github.com/github/vscode-codeql/pull/1007)
 - Allow query result locations with 0 as the end column value. These are treated as the first column in the line. [#1002](https://github.com/github/vscode-codeql/pull/1002)

extensions/ql-vscode/media/globe.svg

@@ -225,6 +225,12 @@
           "default": "%q on %d - %s, %r result count [%t]",
           "markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
         },
+        "codeQL.queryHistory.ttl": {
+          "type": "number",
+          "default": 30,
+          "description": "Number of days to retain queries in the query history before being automatically deleted.",
+          "scope": "machine"
+        },
         "codeQL.runningTests.additionalTestArguments": {
           "scope": "window",
           "type": "array",
@@ -673,7 +679,7 @@
         {
           "command": "codeQLQueryHistory.removeHistoryItem",
           "group": "9_qlCommands",
-          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == cancelledResultsItem"
+          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.setLabel",
@@ -1080,7 +1086,7 @@
     "@types/jszip": "~3.1.6",
     "@types/mocha": "^9.0.0",
     "@types/nanoid": "^3.0.0",
-    "@types/node": "^12.14.1",
+    "@types/node": "^16.11.25",
     "@types/node-fetch": "~2.5.2",
     "@types/proxyquire": "~1.3.28",
     "@types/react": "^17.0.2",

extensions/ql-vscode/package-lock.json

@@ -1,5 +1,5 @@
 /**
- * The jszip and d3 libraries are designed to work in both the browser and
+ * The d3 library is designed to work in both the browser and
  * node. Consequently their typings files refer to both node
  * types like `Buffer` (which don't exist in the browser), and browser
  * types like `Blob` (which don't exist in node). Instead of sticking
@@ -8,7 +8,6 @@
  * succeeds.
  */
 
-declare type Blob = string;
 declare type RequestInit = Record<string, unknown>;
 declare type ElementTagNameMap = any;
 declare type NodeListOf<T> = Record<string, T>;

extensions/ql-vscode/package.json

@@ -404,7 +404,7 @@ export class CodeQLCliServer implements Disposable {
     try {
       if (cancellationToken !== undefined) {
         cancellationRegistration = cancellationToken.onCancellationRequested(_e => {
-          tk(child.pid);
+          tk(child.pid || 0);
         });
       }
       if (logger !== undefined) {
@@ -515,8 +515,7 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(workspaces: string[], queryPath: string): Promise<QuerySetup> {
     const subcommandArgs = [
       '--query', queryPath,
-      '--additional-packs',
-      workspaces.join(path.delimiter)
+      ...this.getAdditionalPacksArg(workspaces)
     ];
     return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, 'Resolving library paths');
   }
@@ -529,8 +528,7 @@ export class CodeQLCliServer implements Disposable {
     const subcommandArgs = [
       '--format', 'bylanguage',
       queryUri.fsPath,
-      '--additional-packs',
-      workspaces.join(path.delimiter)
+      ...this.getAdditionalPacksArg(workspaces)
     ];
     return JSON.parse(await this.runCodeQlCliCommand(['resolve', 'queries'], subcommandArgs, 'Resolving query by language'));
   }
@@ -563,6 +561,17 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Issues an internal clear-cache command to the cli server. This
+   * command is used to clear the qlpack cache of the server.
+   *
+   * This cache is generally cleared every 1s. This method is used
+   * to force an early clearing of the cache.
+   */
+  public async clearCache(): Promise<void> {
+    await this.runCodeQlCliCommand(['clear-cache'], [], 'Clearing qlpack cache');
+  }
+
   /**
    * Runs QL tests.
    * @param testPaths Full paths of the tests to run.
@@ -574,7 +583,7 @@ export class CodeQLCliServer implements Disposable {
   ): AsyncGenerator<TestCompleted, void, unknown> {
 
     const subcommandArgs = this.cliConfig.additionalTestArguments.concat([
-      '--additional-packs', workspaces.join(path.delimiter),
+      ...this.getAdditionalPacksArg(workspaces),
       '--threads',
       this.cliConfig.numberTestThreads.toString(),
       ...testPaths
@@ -596,8 +605,12 @@ export class CodeQLCliServer implements Disposable {
 
   /** Resolves the ML models that should be available when evaluating a query. */
   async resolveMlModels(additionalPacks: string[]): Promise<MlModelsInfo> {
-    return await this.runJsonCodeQlCliCommand<MlModelsInfo>(['resolve', 'ml-models'], ['--additional-packs',
-      additionalPacks.join(path.delimiter)], 'Resolving ML models', false);
+    return await this.runJsonCodeQlCliCommand<MlModelsInfo>(
+      ['resolve', 'ml-models'],
+      this.getAdditionalPacksArg(additionalPacks),
+      'Resolving ML models',
+      false
+    );
   }
 
   /**
@@ -725,9 +738,9 @@ export class CodeQLCliServer implements Disposable {
 
   async interpretBqrsGraph(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<string[]> {
     const additionalArgs = sourceInfo ? ['--dot-location-url-format', 'file://' + sourceInfo.sourceLocationPrefix + '{path}:{start:line}:{start:column}:{end:line}:{end:column}'] : [];
-  
+
     await this.runInterpretCommand('dot', additionalArgs, metadata, resultsPath, interpretedResultsPath, sourceInfo);
-  
+
     try {
       const dot = await this.readDotFiles(interpretedResultsPath);
       return dot;
@@ -783,7 +796,7 @@ export class CodeQLCliServer implements Disposable {
    * @returns A list of database upgrade script directories
    */
   async resolveUpgrades(dbScheme: string, searchPath: string[], allowDowngradesIfPossible: boolean, targetDbScheme?: string): Promise<UpgradesInfo> {
-    const args = ['--additional-packs', searchPath.join(path.delimiter), '--dbscheme', dbScheme];
+    const args = [...this.getAdditionalPacksArg(searchPath), '--dbscheme', dbScheme];
     if (targetDbScheme) {
       args.push('--target-dbscheme', targetDbScheme);
       if (allowDowngradesIfPossible && await this.cliConstraints.supportsDowngrades()) {
@@ -805,7 +818,7 @@ export class CodeQLCliServer implements Disposable {
    * @returns A dictionary mapping qlpack name to the directory it comes from
    */
   resolveQlpacks(additionalPacks: string[], searchPath?: string[]): Promise<QlpacksInfo> {
-    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    const args = this.getAdditionalPacksArg(additionalPacks);
     if (searchPath?.length) {
       args.push('--search-path', path.join(...searchPath));
     }
@@ -851,7 +864,7 @@ export class CodeQLCliServer implements Disposable {
    * @returns A list of query files found.
    */
   async resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
-    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    const args = this.getAdditionalPacksArg(additionalPacks);
     if (searchPath !== undefined) {
       args.push('--search-path', path.join(...searchPath));
     }
@@ -884,8 +897,7 @@ export class CodeQLCliServer implements Disposable {
       '-o',
       outputPath,
       dir,
-      '--additional-packs',
-      workspaceFolders.join(path.delimiter)
+      ...this.getAdditionalPacksArg(workspaceFolders)
     ];
     if (!precompile && await this.cliConstraints.supportsNoPrecompile()) {
       args.push('--no-precompile');
@@ -940,6 +952,12 @@ export class CodeQLCliServer implements Disposable {
         throw new Error('No distribution found');
     }
   }
+
+  private getAdditionalPacksArg(paths: string[]): string[] {
+    return paths.length
+      ? ['--additional-packs', paths.join(path.delimiter)]
+      : [];
+  }
 }
 
 /**
@@ -1226,6 +1244,12 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_PACKAGING = new SemVer('2.6.0');
 
+  /**
+   * CLI version where the `--evaluator-log` and related options to the query server were introduced,
+   * on a per-query server basis. 
+   */
+  public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer('2.8.2');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1281,4 +1305,8 @@ export class CliVersionConstraint {
   async supportsPackaging() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PACKAGING);
   }
+
+  async supportsStructuredEvalLog() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG);
+  }
 }

extensions/ql-vscode/src/additional-typings.d.ts

@@ -160,7 +160,8 @@ export function commandRunner(
 export function commandRunnerWithProgress<R>(
   commandId: string,
   task: ProgressTask<R>,
-  progressOptions: Partial<ProgressOptions>
+  progressOptions: Partial<ProgressOptions>,
+  outputLogger = logger
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
     const startTime = Date.now();
@@ -177,16 +178,17 @@ export function commandRunnerWithProgress<R>(
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          void logger.log(errorMessage);
+          void outputLogger.log(errorMessage);
         } else {
-          void showAndLogWarningMessage(errorMessage);
+          void showAndLogWarningMessage(errorMessage, { outputLogger });
         }
       } else {
         // Include the full stack in the error log only.
         const fullMessage = e.stack
           ? `${errorMessage}\n${e.stack}`
           : errorMessage;
         void showAndLogErrorMessage(errorMessage, {
+          outputLogger,
           fullMessage
         });
       }