QueryServer: Add support for new query-server (#1508)

* QueryServer: Add support for new query-server

* Add a new canary flag to enable new query server support

* Add evaluation results to query object

Ensures better backwards compatibility with legacy query objects.

* Fix query server command name

* Add log message for new query server

* Use only legacy results

Co-authored-by: alexet <alexet@semmle.com>

Author: aeisenberg

extensions/ql-vscode/src/cli.ts

@@ -11,7 +11,7 @@ import { promisify } from 'util';
 import { CancellationToken, commands, Disposable, Uri } from 'vscode';
 
 import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
-import { CliConfig } from './config';
+import { allowCanaryQueryServer, CliConfig } from './config';
 import { DistributionProvider, FindDistributionResultKind } from './distribution';
 import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import { QueryMetadata, SortDirection } from './pure/interface-types';
@@ -1330,6 +1330,11 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_SOURCEMAP = new SemVer('2.10.3');
 
+  /**
+   * CLI version that supports the new query server.
+   */
+  public static CLI_VERSION_WITH_NEW_QUERY_SERVER = new SemVer('2.11.0');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1405,4 +1410,12 @@ export class CliVersionConstraint {
   async supportsSourceMap() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_SOURCEMAP);
   }
+
+  async supportsNewQueryServer() {
+    // TODO while under development, users _must_ opt-in to the new query server
+    // by setting the `codeql.canaryQueryServer` setting to `true`.
+    // Ignore the version check for now.
+    return allowCanaryQueryServer();
+    // return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NEW_QUERY_SERVER);
+  }
 }

extensions/ql-vscode/src/config.ts

@@ -317,6 +317,17 @@ export function isCanary() {
   return !!CANARY_FEATURES.getValue<boolean>();
 }
 
+/**
+ * Enables the experimental query server
+ */
+export const CANARY_QUERY_SERVER = new Setting('canaryQueryServer', ROOT_SETTING);
+
+
+export function allowCanaryQueryServer() {
+  return !!CANARY_QUERY_SERVER.getValue<boolean>();
+}
+
+
 /**
  * Avoids caching in the AST viewer if the user is also a canary user.
  */
@@ -343,12 +354,12 @@ export async function setRemoteRepositoryLists(lists: Record<string, string[]> |
 }
 
 /**
- * Path to a file that contains lists of GitHub repositories that you want to query remotely via 
+ * Path to a file that contains lists of GitHub repositories that you want to query remotely via
  * the "Run Variant Analysis" command.
  * Note: This command is only available for internal users.
- * 
+ *
  * This setting should be a path to a JSON file that contains a JSON object where each key is a
- * user-specified name (string), and the value is an array of GitHub repositories 
+ * user-specified name (string), and the value is an array of GitHub repositories
  * (of the form `<owner>/<repo>`).
  */
 const REPO_LISTS_PATH = new Setting('repositoryListsPath', REMOTE_QUERIES_SETTING);

extensions/ql-vscode/src/extension.ts

@@ -73,7 +73,8 @@ import { WebviewReveal } from './interface-utils';
 import { ideServerLogger, logger, ProgressReporter, queryServerLogger } from './logging';
 import { QueryHistoryManager } from './query-history';
 import { CompletedLocalQueryInfo, LocalQueryInfo } from './query-results';
-import * as qsClient from './legacy-query-server/queryserver-client';
+import * as legacyQueryServer from './legacy-query-server/queryserver-client';
+import * as newQueryServer from './query-server/queryserver-client';
 import { displayQuickQuery } from './quick-query';
 import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
@@ -103,6 +104,7 @@ import { JoinOrderScannerProvider } from './log-insights/join-order';
 import { LogScannerService } from './log-insights/log-scanner-service';
 import { createInitialQueryInfo } from './run-queries-shared';
 import { LegacyQueryRunner } from './legacy-query-server/legacyRunner';
+import { NewQueryRunner } from './query-server/query-runner';
 import { QueryRunner } from './queryRunner';
 import { VariantAnalysisView } from './remote-queries/variant-analysis-view';
 import { VariantAnalysisViewSerializer } from './remote-queries/variant-analysis-view-serializer';
@@ -406,6 +408,8 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
   return codeQlExtension;
 }
 
+const PACK_GLOBS = ['**/codeql-pack.yml', '**/qlpack.yml', '**/queries.xml', '**/codeql-pack.lock.yml', '**/qlpack.lock.yml', '.codeqlmanifest.json', 'codeql-workspace.yml'];
+
 async function activateWithInstalledDistribution(
   ctx: ExtensionContext,
   distributionManager: DistributionManager,
@@ -436,6 +440,15 @@ async function activateWithInstalledDistribution(
   void logger.log('Initializing query server client.');
   const qs = await createQueryServer(qlConfigurationListener, cliServer, ctx);
 
+
+  for (const glob of PACK_GLOBS) {
+    const fsWatcher = workspace.createFileSystemWatcher(glob);
+    ctx.subscriptions.push(fsWatcher);
+    fsWatcher.onDidChange(async (_uri) => {
+      await qs.clearPackCache();
+    });
+  }
+
   void logger.log('Initializing database manager.');
   const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
   ctx.subscriptions.push(dbm);
@@ -1199,15 +1212,28 @@ async function createQueryServer(qlConfigurationListener: QueryServerConfigListe
     { title: 'CodeQL query server', location: ProgressLocation.Window },
     task
   );
-  const qs = new qsClient.QueryServerClient(
-    qlConfigurationListener,
-    cliServer,
-    qsOpts,
-    progressCallback
-  );
-  ctx.subscriptions.push(qs);
-  await qs.startQueryServer();
-  return new LegacyQueryRunner(qs);
+  if (await cliServer.cliConstraints.supportsNewQueryServer()) {
+    const qs = new newQueryServer.QueryServerClient(
+      qlConfigurationListener,
+      cliServer,
+      qsOpts,
+      progressCallback
+    );
+    ctx.subscriptions.push(qs);
+    await qs.startQueryServer();
+    return new NewQueryRunner(qs);
+
+  } else {
+    const qs = new legacyQueryServer.QueryServerClient(
+      qlConfigurationListener,
+      cliServer,
+      qsOpts,
+      progressCallback
+    );
+    ctx.subscriptions.push(qs);
+    await qs.startQueryServer();
+    return new LegacyQueryRunner(qs);
+  }
 }
 
 function getContextStoragePath(ctx: ExtensionContext) {

extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts

@@ -56,4 +56,10 @@ export class LegacyQueryRunner extends QueryRunner {
   async upgradeDatabaseExplicit(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
     await upgradeDatabaseExplicit(this.qs, dbItem, progress, token);
   }
+
+  async clearPackCache(): Promise<void> {
+    /**
+     * Nothing needs to be done
+     */
+  }
 }

extensions/ql-vscode/src/legacy-query-server/run-queries.ts

@@ -497,11 +497,19 @@ function createSyntheticResult(
   return {
     query: query.queryEvalInfo,
     message,
+    result:{
+      evaluationTime:0,
+      queryId: 0,
+      resultType: messages.QueryResultType.OTHER_ERROR,
+      message,
+      runId: 0,
+    },
     sucessful: false,
     dispose: () => { /**/ },
   };
 }
 
+
 function createSimpleTemplates(templates: Record<string, string> | undefined): messages.TemplateDefinitions | undefined {
   if (!templates) {
     return undefined;

extensions/ql-vscode/src/legacy-query-server/upgrades.ts

@@ -3,7 +3,7 @@ import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from '../he
 import { ProgressCallback, UserCancellationException } from '../commandRunner';
 import { logger } from '../logging';
 import * as messages from '../pure/legacy-messages';
-import * as qsClient from '../legacy-query-server/queryserver-client';
+import * as qsClient from './queryserver-client';
 import * as tmp from 'tmp-promise';
 import * as path from 'path';
 import { DatabaseItem } from '../databases';