Add support for system defined repository lists (#1271)

Author: charisk

extensions/ql-vscode/src/remote-queries/repository-selection.ts

@@ -1,54 +1,116 @@
 import { QuickPickItem, window } from 'vscode';
 import { showAndLogErrorMessage } from '../helpers';
-import { getRemoteRepositoryLists } from '../config';
 import { logger } from '../logging';
+import { getRemoteRepositoryLists } from '../config';
 import { REPO_REGEX } from '../pure/helpers-pure';
 
+export interface RepositorySelection {
+  repositories?: string[];
+  repositoryLists?: string[]
+}
+
 interface RepoListQuickPickItem extends QuickPickItem {
-  repoList: string[];
+  repositories?: string[];
+  repositoryList?: string;
+  useCustomRepository?: boolean;
 }
 
 /**
- * Gets the repositories to run the query against.
+ * Gets the repositories or repository lists to run the query against.
+ * @returns The user selection.
  */
-export async function getRepositories(): Promise<string[] | undefined> {
-  const repoLists = getRemoteRepositoryLists();
-  if (repoLists && Object.keys(repoLists).length) {
-    const quickPickItems = Object.entries(repoLists).map<RepoListQuickPickItem>(([key, value]) => (
-      {
-        label: key,       // the name of the repository list
-        repoList: value,  // the actual array of repositories
-      }
-    ));
-    const quickpick = await window.showQuickPick<RepoListQuickPickItem>(
-      quickPickItems,
-      {
-        placeHolder: 'Select a repository list. You can define repository lists in the `codeQL.variantAnalysis.repositoryLists` setting.',
-        ignoreFocusOut: true,
-      });
-    if (quickpick?.repoList.length) {
-      void logger.log(`Selected repositories: ${quickpick.repoList.join(', ')}`);
-      return quickpick.repoList;
-    } else {
-      void showAndLogErrorMessage('No repositories selected.');
-      return;
+export async function getRepositorySelection(): Promise<RepositorySelection> {
+  const quickPickItems = [
+    createCustomRepoQuickPickItem(),
+    ...createSystemDefinedRepoListsQuickPickItems(),
+    ...createUserDefinedRepoListsQuickPickItems(),
+  ];
+
+  const options = {
+    placeHolder: 'Select a repository list. You can define repository lists in the `codeQL.variantAnalysis.repositoryLists` setting.',
+    ignoreFocusOut: true,
+  };
+
+  const quickpick = await window.showQuickPick<RepoListQuickPickItem>(
+    quickPickItems,
+    options);
+
+  if (quickpick?.repositories?.length) {
+    void logger.log(`Selected repositories: ${quickpick.repositories.join(', ')}`);
+    return { repositories: quickpick.repositories };
+  } else if (quickpick?.repositoryList) {
+    void logger.log(`Selected repository list: ${quickpick.repositoryList}`);
+    return { repositoryLists: [quickpick.repositoryList] };
+  } else if (quickpick?.useCustomRepository) {
+    const customRepo = await getCustomRepo();
+    if (!customRepo || !REPO_REGEX.test(customRepo)) {
+      void showAndLogErrorMessage('Invalid repository format. Please enter a valid repository in the format <owner>/<repo> (e.g. github/codeql)');
+      return {};
     }
+    void logger.log(`Entered repository: ${customRepo}`);
+    return { repositories: [customRepo] };
   } else {
-    void logger.log('No repository lists defined. Displaying text input box.');
-    const remoteRepo = await window.showInputBox({
-      title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
-      placeHolder: '<owner>/<repo>',
-      prompt: 'Tip: you can save frequently used repositories in the `codeQL.variantAnalysis.repositoryLists` setting',
-      ignoreFocusOut: true,
-    });
-    if (!remoteRepo) {
-      void showAndLogErrorMessage('No repositories entered.');
-      return;
-    } else if (!REPO_REGEX.test(remoteRepo)) { // Check if user entered invalid input
-      void showAndLogErrorMessage('Invalid repository format. Must be in the format <owner>/<repo> (e.g. github/codeql)');
-      return;
-    }
-    void logger.log(`Entered repository: ${remoteRepo}`);
-    return [remoteRepo];
+    void showAndLogErrorMessage('No repositories selected.');
+    return {};
+  }
+}
+
+/**
+ * Checks if the selection is valid or not.
+ * @param repoSelection The selection to check.
+ * @returns A boolean flag indicating if the selection is valid or not.
+ */
+export function isValidSelection(repoSelection: RepositorySelection): boolean {
+  if (repoSelection.repositories === undefined && repoSelection.repositoryLists === undefined) {
+    return false;
+  }
+  if (repoSelection.repositories !== undefined && repoSelection.repositories.length === 0) {
+    return false;
   }
+  if (repoSelection.repositoryLists?.length === 0) {
+    return false;
+  }
+
+  return true;
+}
+
+function createSystemDefinedRepoListsQuickPickItems(): RepoListQuickPickItem[] {
+  const topNs = [10, 100, 1000];
+
+  return topNs.map(n => ({
+    label: '$(star) Top ' + n,
+    repositoryList: `top_${n}`,
+    alwaysShow: true
+  } as RepoListQuickPickItem));
+}
+
+function createUserDefinedRepoListsQuickPickItems(): RepoListQuickPickItem[] {
+  const repoLists = getRemoteRepositoryLists();
+  if (!repoLists) {
+    return [];
+  }
+
+  return Object.entries(repoLists).map<RepoListQuickPickItem>(([label, repositories]) => (
+    {
+      label,            // the name of the repository list
+      repositories  // the actual array of repositories
+    }
+  ));
+}
+
+function createCustomRepoQuickPickItem(): RepoListQuickPickItem {
+  return {
+    label: '$(edit) Enter a GitHub repository',
+    useCustomRepository: true,
+    alwaysShow: true,
+  };
+}
+
+async function getCustomRepo(): Promise<string | undefined> {
+  return await window.showInputBox({
+    title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
+    placeHolder: '<owner>/<repo>',
+    prompt: 'Tip: you can save frequently used repositories in the `codeQL.variantAnalysis.repositoryLists` setting',
+    ignoreFocusOut: true,
+  });
 }

extensions/ql-vscode/src/remote-queries/run-remote-query.ts

@@ -22,7 +22,7 @@ import { RemoteQuery } from './remote-query';
 import { RemoteQuerySubmissionResult } from './remote-query-submission-result';
 import { QueryMetadata } from '../pure/interface-types';
 import { getErrorMessage, REPO_REGEX } from '../pure/helpers-pure';
-import { getRepositories } from './repository-selection';
+import { getRepositorySelection, isValidSelection, RepositorySelection } from './repository-selection';
 
 export interface QlPack {
   name: string;
@@ -189,8 +189,8 @@ export async function runRemoteQuery(
       message: 'Determining query target language'
     });
 
-    const repositories = await getRepositories();
-    if (!repositories || repositories.length === 0) {
+    const repoSelection = await getRepositorySelection();
+    if (!isValidSelection(repoSelection)) {
       throw new UserCancellationException('No repositories to query.');
     }
 
@@ -249,7 +249,7 @@ export async function runRemoteQuery(
     });
 
     const actionBranch = getActionBranch();
-    const workflowRunId = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repositories, owner, repo, base64Pack, dryRun);
+    const workflowRunId = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, owner, repo, base64Pack, dryRun);
     const queryStartTime = Date.now();
     const queryMetadata = await tryGetQueryMetadata(cliServer, queryFile);
 
@@ -287,15 +287,30 @@ async function runRemoteQueriesApiRequest(
   credentials: Credentials,
   ref: string,
   language: string,
-  repositories: string[],
+  repoSelection: RepositorySelection,
   owner: string,
   repo: string,
   queryPackBase64: string,
   dryRun = false
 ): Promise<void | number> {
+  const data = {
+    ref,
+    language,
+    repositories: repoSelection.repositories ?? undefined,
+    repository_lists: repoSelection.repositoryLists ?? undefined,
+    query_pack: queryPackBase64,
+  };
+
   if (dryRun) {
     void showAndLogInformationMessage('[DRY RUN] Would have sent request. See extension log for the payload.');
-    void logger.log(JSON.stringify({ ref, language, repositories, owner, repo, queryPackBase64: queryPackBase64.substring(0, 100) + '... ' + queryPackBase64.length + ' bytes' }));
+    void logger.log(JSON.stringify({
+      owner,
+      repo,
+      data: {
+        ...data,
+        queryPackBase64: queryPackBase64.substring(0, 100) + '... ' + queryPackBase64.length + ' bytes'
+      }
+    }));
     return;
   }
 
@@ -306,12 +321,7 @@ async function runRemoteQueriesApiRequest(
       {
         owner,
         repo,
-        data: {
-          ref,
-          language,
-          repositories,
-          query_pack: queryPackBase64,
-        }
+        data
       }
     );
     const workflowRunId = response.data.workflow_run_id;

extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/run-remote-query.test.ts

@@ -52,7 +52,7 @@ describe('Remote queries', function() {
     progress = sandbox.spy();
     // Should not have asked for a language
     showQuickPickSpy = sandbox.stub(window, 'showQuickPick')
-      .onFirstCall().resolves({ repoList: ['github/vscode-codeql'] } as unknown as QuickPickItem)
+      .onFirstCall().resolves({ repositories: ['github/vscode-codeql'] } as unknown as QuickPickItem)
       .onSecondCall().resolves('javascript' as unknown as QuickPickItem);
 
     // always run in the vscode-codeql repo

extensions/ql-vscode/src/vscode-tests/no-workspace/remote-queries/repository-selection.test.ts

@@ -8,7 +8,7 @@ const proxyquire = pq.noPreserveCache();
 
 describe('repository-selection', function() {
 
-  describe('getRepositories', () => {
+  describe('getRepositorySelection', () => {
     let sandbox: sinon.SinonSandbox;
     let quickPickSpy: sinon.SinonStub;
     let showInputBoxSpy: sinon.SinonStub;
@@ -35,10 +35,10 @@ describe('repository-selection', function() {
       sandbox.restore();
     });
 
-    it('should run on a repo list that you chose from your pre-defined config', async () => {
+    it('should allow selection from repo lists from your pre-defined config', async () => {
       // fake return values
       quickPickSpy.resolves(
-        { repoList: ['foo/bar', 'foo/baz'] }
+        { repositories: ['foo/bar', 'foo/baz'] }
       );
       getRemoteRepositoryListsSpy.returns(
         {
@@ -48,14 +48,37 @@ describe('repository-selection', function() {
       );
 
       // make the function call
-      const repoList = await mod.getRepositories();
+      const repoSelection = await mod.getRepositorySelection();
 
       // Check that the return value is correct
-      expect(repoList).to.deep.eq(
+      expect(repoSelection.repositoryLists).to.be.undefined;
+      expect(repoSelection.repositories).to.deep.eq(
         ['foo/bar', 'foo/baz']
       );
     });
 
+    it('should allow selection from repo lists defined at the system level', async () => {
+      // fake return values
+      quickPickSpy.resolves(
+        { repositoryList: 'top_100' }
+      );
+      getRemoteRepositoryListsSpy.returns(
+        {
+          'list1': ['foo/bar', 'foo/baz'],
+          'list2': [],
+        }
+      );
+
+      // make the function call
+      const repoSelection = await mod.getRepositorySelection();
+
+      // Check that the return value is correct
+      expect(repoSelection.repositories).to.be.undefined;
+      expect(repoSelection.repositoryLists).to.deep.eq(
+        ['top_100']
+      );
+    });
+
     // Test the regex in various "good" cases
     const goodRepos = [
       'owner/repo',
@@ -65,14 +88,17 @@ describe('repository-selection', function() {
     goodRepos.forEach(repo => {
       it(`should run on a valid repo that you enter in the text box: ${repo}`, async () => {
         // fake return values
+        quickPickSpy.resolves(
+          { useCustomRepository: true }
+        );
         getRemoteRepositoryListsSpy.returns({}); // no pre-defined repo lists
         showInputBoxSpy.resolves(repo);
 
         // make the function call
-        const repoList = await mod.getRepositories();
+        const repoSelection = await mod.getRepositorySelection();
 
         // Check that the return value is correct
-        expect(repoList).to.deep.equal(
+        expect(repoSelection.repositories).to.deep.equal(
           [repo]
         );
       });
@@ -88,11 +114,14 @@ describe('repository-selection', function() {
     badRepos.forEach(repo => {
       it(`should show an error message if you enter an invalid repo in the text box: ${repo}`, async () => {
         // fake return values
+        quickPickSpy.resolves(
+          { useCustomRepository: true }
+        );
         getRemoteRepositoryListsSpy.returns({}); // no pre-defined repo lists
         showInputBoxSpy.resolves(repo);
 
         // make the function call
-        await mod.getRepositories();
+        await mod.getRepositorySelection();
 
         // check that we get the right error message
         expect(showAndLogErrorMessageSpy.firstCall.args[0]).to.contain('Invalid repository format');