Move walkDirectory to pure files file

Author: koesie10

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -22,11 +22,11 @@ import {
   getErrorMessage,
   getErrorStack,
 } from "../pure/helpers-pure";
+import { walkDirectory } from "../pure/files";
 import { QueryMetadata, SortDirection } from "../pure/interface-types";
 import { BaseLogger, Logger, ProgressReporter } from "../common";
 import { CompilationMessage } from "../pure/legacy-messages";
 import { sarifParser } from "../common/sarif-parser";
-import { walkDirectory } from "../helpers";
 import { App } from "../common/app";
 import { QueryLanguage } from "../common/query-language";
 

extensions/ql-vscode/src/helpers.ts

@@ -1,10 +1,4 @@
-import {
-  ensureDirSync,
-  pathExists,
-  ensureDir,
-  writeFile,
-  opendir,
-} from "fs-extra";
+import { ensureDirSync, pathExists, ensureDir, writeFile } from "fs-extra";
 import { join, dirname } from "path";
 import { dirSync } from "tmp-promise";
 import { Uri, window as Window, workspace, env, WorkspaceFolder } from "vscode";
@@ -455,29 +449,6 @@ export async function createTimestampFile(storagePath: string) {
   await writeFile(timestampPath, Date.now().toString(), "utf8");
 }
 
-/**
- * Recursively walk a directory and return the full path to all files found.
- * Symbolic links are ignored.
- *
- * @param dir the directory to walk
- *
- * @return An iterator of the full path to all files recursively found in the directory.
- */
-export async function* walkDirectory(
-  dir: string,
-): AsyncIterableIterator<string> {
-  const seenFiles = new Set<string>();
-  for await (const d of await opendir(dir)) {
-    const entry = join(dir, d.name);
-    seenFiles.add(entry);
-    if (d.isDirectory()) {
-      yield* walkDirectory(entry);
-    } else if (d.isFile()) {
-      yield entry;
-    }
-  }
-}
-
 /**
  * Returns the path of the first folder in the workspace.
  * This is used to decide where to create skeleton QL packs.

extensions/ql-vscode/src/pure/files.ts

@@ -1,4 +1,4 @@
-import { pathExists, stat, readdir } from "fs-extra";
+import { pathExists, stat, readdir, opendir } from "fs-extra";
 import { join, resolve } from "path";
 
 /**
@@ -88,3 +88,26 @@ export async function readDirFullPaths(path: string): Promise<string[]> {
   const baseNames = await readdir(path);
   return baseNames.map((baseName) => join(path, baseName));
 }
+
+/**
+ * Recursively walk a directory and return the full path to all files found.
+ * Symbolic links are ignored.
+ *
+ * @param dir the directory to walk
+ *
+ * @return An iterator of the full path to all files recursively found in the directory.
+ */
+export async function* walkDirectory(
+  dir: string,
+): AsyncIterableIterator<string> {
+  const seenFiles = new Set<string>();
+  for await (const d of await opendir(dir)) {
+    const entry = join(dir, d.name);
+    seenFiles.add(entry);
+    if (d.isDirectory()) {
+      yield* walkDirectory(entry);
+    } else if (d.isFile()) {
+      yield entry;
+    }
+  }
+}

extensions/ql-vscode/test/unit-tests/pure/files.test.ts

@@ -6,7 +6,11 @@ import {
   getDirectoryNamesInsidePath,
   pathsEqual,
   readDirFullPaths,
+  walkDirectory,
 } from "../../../src/pure/files";
+import { DirResult } from "tmp";
+import * as tmp from "tmp";
+import { ensureDirSync, symlinkSync, writeFileSync } from "fs-extra";
 
 describe("files", () => {
   const dataDir = join(__dirname, "../../data");
@@ -299,3 +303,67 @@ describe("containsPath", () => {
     },
   );
 });
+
+describe("walkDirectory", () => {
+  let tmpDir: DirResult;
+  let dir: string;
+  let dir2: string;
+
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    dir = join(tmpDir.name, "dir");
+    ensureDirSync(dir);
+    dir2 = join(tmpDir.name, "dir2");
+  });
+
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("should walk a directory", async () => {
+    const file1 = join(dir, "file1");
+    const file2 = join(dir, "file2");
+    const file3 = join(dir, "file3");
+    const dir3 = join(dir, "dir3");
+    const file4 = join(dir, "file4");
+    const file5 = join(dir, "file5");
+    const file6 = join(dir, "file6");
+
+    // These symlinks link back to paths that are already existing, so ignore.
+    const symLinkFile7 = join(dir, "symlink0");
+    const symlinkDir = join(dir2, "symlink1");
+
+    // some symlinks that point outside of the base dir.
+    const file8 = join(tmpDir.name, "file8");
+    const file9 = join(dir2, "file8");
+    const symlinkDir2 = join(dir2, "symlink2");
+    const symlinkFile2 = join(dir2, "symlinkFile3");
+
+    ensureDirSync(dir2);
+    ensureDirSync(dir3);
+
+    writeFileSync(file1, "file1");
+    writeFileSync(file2, "file2");
+    writeFileSync(file3, "file3");
+    writeFileSync(file4, "file4");
+    writeFileSync(file5, "file5");
+    writeFileSync(file6, "file6");
+    writeFileSync(file8, "file8");
+    writeFileSync(file9, "file9");
+
+    // We don't really need to be testing all of these variants of symlinks,
+    // but it doesn't hurt, and will help us if we ever do decide to support them.
+    symlinkSync(file6, symLinkFile7, "file");
+    symlinkSync(dir3, symlinkDir, "dir");
+    symlinkSync(file8, symlinkFile2, "file");
+    symlinkSync(dir2, symlinkDir2, "dir");
+
+    const files = [];
+    for await (const file of walkDirectory(dir)) {
+      files.push(file);
+    }
+
+    // Only real files should be returned.
+    expect(files.sort()).toEqual([file1, file2, file3, file4, file5, file6]);
+  });
+});

extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts

@@ -1,14 +1,7 @@
 import { Uri, window, workspace, WorkspaceFolder } from "vscode";
 import * as tmp from "tmp";
 import { join } from "path";
-import {
-  writeFileSync,
-  ensureDirSync,
-  symlinkSync,
-  writeFile,
-  mkdir,
-} from "fs-extra";
-import { DirResult } from "tmp";
+import { writeFile, mkdir } from "fs-extra";
 
 import {
   getFirstWorkspaceFolder,
@@ -18,7 +11,6 @@ import {
   showBinaryChoiceWithUrlDialog,
   showInformationMessageWithAction,
   showNeverAskAgainDialog,
-  walkDirectory,
 } from "../../../src/helpers";
 import { reportStreamProgress } from "../../../src/common/vscode/progress";
 import { Setting } from "../../../src/config";
@@ -243,70 +235,6 @@ describe("helpers", () => {
   });
 });
 
-describe("walkDirectory", () => {
-  let tmpDir: DirResult;
-  let dir: string;
-  let dir2: string;
-
-  beforeEach(() => {
-    tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    dir = join(tmpDir.name, "dir");
-    ensureDirSync(dir);
-    dir2 = join(tmpDir.name, "dir2");
-  });
-
-  afterEach(() => {
-    tmpDir.removeCallback();
-  });
-
-  it("should walk a directory", async () => {
-    const file1 = join(dir, "file1");
-    const file2 = join(dir, "file2");
-    const file3 = join(dir, "file3");
-    const dir3 = join(dir, "dir3");
-    const file4 = join(dir, "file4");
-    const file5 = join(dir, "file5");
-    const file6 = join(dir, "file6");
-
-    // These symlinks link back to paths that are already existing, so ignore.
-    const symLinkFile7 = join(dir, "symlink0");
-    const symlinkDir = join(dir2, "symlink1");
-
-    // some symlinks that point outside of the base dir.
-    const file8 = join(tmpDir.name, "file8");
-    const file9 = join(dir2, "file8");
-    const symlinkDir2 = join(dir2, "symlink2");
-    const symlinkFile2 = join(dir2, "symlinkFile3");
-
-    ensureDirSync(dir2);
-    ensureDirSync(dir3);
-
-    writeFileSync(file1, "file1");
-    writeFileSync(file2, "file2");
-    writeFileSync(file3, "file3");
-    writeFileSync(file4, "file4");
-    writeFileSync(file5, "file5");
-    writeFileSync(file6, "file6");
-    writeFileSync(file8, "file8");
-    writeFileSync(file9, "file9");
-
-    // We don't really need to be testing all of these variants of symlinks,
-    // but it doesn't hurt, and will help us if we ever do decide to support them.
-    symlinkSync(file6, symLinkFile7, "file");
-    symlinkSync(dir3, symlinkDir, "dir");
-    symlinkSync(file8, symlinkFile2, "file");
-    symlinkSync(dir2, symlinkDir2, "dir");
-
-    const files = [];
-    for await (const file of walkDirectory(dir)) {
-      files.push(file);
-    }
-
-    // Only real files should be returned.
-    expect(files.sort()).toEqual([file1, file2, file3, file4, file5, file6]);
-  });
-});
-
 describe("isFolderAlreadyInWorkspace", () => {
   beforeEach(() => {
     const folders = [

extensions/ql-vscode/test/vscode-tests/no-workspace/query-history/variant-analysis-history.test.ts

@@ -10,7 +10,8 @@ import { join } from "path";
 
 import { ExtensionContext, Uri } from "vscode";
 import { DatabaseManager } from "../../../../src/databases/local-databases";
-import { tmpDir, walkDirectory } from "../../../../src/helpers";
+import { tmpDir } from "../../../../src/helpers";
+import { walkDirectory } from "../../../../src/pure/files";
 import { DisposableBucket } from "../../disposable-bucket";
 import { testDisposeHandler } from "../../test-dispose-handler";
 import { HistoryItemLabelProvider } from "../../../../src/query-history/history-item-label-provider";