Skip to content

Commit 5098382

Browse files
koesie10koesie10
authored
Merge pull request #2014 from github/koesie10/download-progress
Report download progress when downloading variant analysis results
2 parents 5ca4097 + 0efbbbf commit 5098382

13 files changed

Lines changed: 192 additions & 106 deletions

File tree

‎extensions/ql-vscode/src/extension.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -633,7 +633,6 @@ async function activateWithInstalledDistribution(
633633
);
634634
await ensureDir(variantAnalysisStorageDir);
635635
const variantAnalysisResultsManager = new VariantAnalysisResultsManager(
636-
app.credentials,
637636
cliServer,
638637
extLogger,
639638
);

‎extensions/ql-vscode/src/remote-queries/gh-api/gh-api-client.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -81,16 +81,6 @@ export async function getVariantAnalysisRepo(
8181
return response.data;
8282
}
8383

84-
export async function getVariantAnalysisRepoResult(
85-
credentials: Credentials,
86-
downloadUrl: string,
87-
): Promise<ArrayBuffer> {
88-
const octokit = await credentials.getOctokit();
89-
const response = await octokit.request(`GET ${downloadUrl}`);
90-
91-
return response.data;
92-
}
93-
9484
export async function getRepositoryFromNwo(
9585
credentials: Credentials,
9686
owner: string,

‎extensions/ql-vscode/src/remote-queries/shared/variant-analysis.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ export enum VariantAnalysisScannedRepositoryDownloadStatus {
127127
export interface VariantAnalysisScannedRepositoryState {
128128
repositoryId: number;
129129
downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus;
130+
downloadPercentage?: number;
130131
}
131132

132133
export interface VariantAnalysisScannedRepositoryResult {

‎extensions/ql-vscode/src/remote-queries/variant-analysis-manager.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ export class VariantAnalysisManager
6868
implements VariantAnalysisViewManager<VariantAnalysisView>
6969
{
7070
private static readonly REPO_STATES_FILENAME = "repo_states.json";
71+
private static readonly DOWNLOAD_PERCENTAGE_UPDATE_DELAY_MS = 3000;
7172

7273
private readonly _onVariantAnalysisAdded = this.push(
7374
new EventEmitter<VariantAnalysis>(),
@@ -514,10 +515,27 @@ export class VariantAnalysisManager
514515
await this.onRepoStateUpdated(variantAnalysis.id, repoState);
515516

516517
try {
518+
let lastRepoStateUpdate = 0;
519+
const updateRepoStateCallback = async (downloadPercentage: number) => {
520+
const now = new Date().getTime();
521+
if (
522+
lastRepoStateUpdate <
523+
now - VariantAnalysisManager.DOWNLOAD_PERCENTAGE_UPDATE_DELAY_MS
524+
) {
525+
lastRepoStateUpdate = now;
526+
await this.onRepoStateUpdated(variantAnalysis.id, {
527+
repositoryId: scannedRepo.repository.id,
528+
downloadStatus:
529+
VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
530+
downloadPercentage,
531+
});
532+
}
533+
};
517534
await this.variantAnalysisResultsManager.download(
518535
variantAnalysis.id,
519536
repoTask,
520537
this.getVariantAnalysisStorageLocation(variantAnalysis.id),
538+
updateRepoStateCallback,
521539
);
522540
} catch (e) {
523541
repoState.downloadStatus =

‎extensions/ql-vscode/src/remote-queries/variant-analysis-results-manager.ts‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,8 @@
1-
import {
2-
pathExists,
3-
mkdir,
4-
outputJson,
5-
writeFileSync,
6-
readJson,
7-
} from "fs-extra";
1+
import { appendFile, pathExists, mkdir, outputJson, readJson } from "fs-extra";
2+
import fetch from "node-fetch";
83
import { EOL } from "os";
94
import { join } from "path";
105

11-
import { Credentials } from "../common/authentication";
126
import { Logger } from "../common";
137
import { AnalysisAlert, AnalysisRawResults } from "./shared/analysis-result";
148
import { sarifParser } from "../sarif-parser";
@@ -21,7 +15,6 @@ import {
2115
VariantAnalysisScannedRepositoryResult,
2216
} from "./shared/variant-analysis";
2317
import { DisposableObject, DisposeHandler } from "../pure/disposable-object";
24-
import { getVariantAnalysisRepoResult } from "./gh-api/gh-api-client";
2518
import { EventEmitter } from "vscode";
2619
import { unzipFile } from "../pure/zip";
2720

@@ -63,7 +56,6 @@ export class VariantAnalysisResultsManager extends DisposableObject {
6356
readonly onResultLoaded = this._onResultLoaded.event;
6457

6558
constructor(
66-
private readonly credentials: Credentials,
6759
private readonly cliServer: CodeQLCliServer,
6860
private readonly logger: Logger,
6961
) {
@@ -75,6 +67,7 @@ export class VariantAnalysisResultsManager extends DisposableObject {
7567
variantAnalysisId: number,
7668
repoTask: VariantAnalysisRepositoryTask,
7769
variantAnalysisStoragePath: string,
70+
onDownloadPercentageChanged: (downloadPercentage: number) => Promise<void>,
7871
): Promise<void> {
7972
if (!repoTask.artifactUrl) {
8073
throw new Error("Missing artifact URL");
@@ -85,11 +78,6 @@ export class VariantAnalysisResultsManager extends DisposableObject {
8578
repoTask.repository.fullName,
8679
);
8780

88-
const result = await getVariantAnalysisRepoResult(
89-
this.credentials,
90-
repoTask.artifactUrl,
91-
);
92-
9381
if (!(await pathExists(resultDirectory))) {
9482
await mkdir(resultDirectory, { recursive: true });
9583
}
@@ -100,12 +88,22 @@ export class VariantAnalysisResultsManager extends DisposableObject {
10088
);
10189

10290
const zipFilePath = join(resultDirectory, "results.zip");
91+
92+
const response = await fetch(repoTask.artifactUrl);
93+
let amountDownloaded = 0;
94+
for await (const chunk of response.body) {
95+
await appendFile(zipFilePath, Buffer.from(chunk));
96+
amountDownloaded += chunk.length;
97+
await onDownloadPercentageChanged(
98+
Math.floor((amountDownloaded / response.size) * 100),
99+
);
100+
}
101+
103102
const unzippedFilesDirectory = join(
104103
resultDirectory,
105104
VariantAnalysisResultsManager.RESULTS_DIRECTORY,
106105
);
107106

108-
writeFileSync(zipFilePath, Buffer.from(result));
109107
await unzipFile(zipFilePath, unzippedFilesDirectory);
110108

111109
this._onResultDownloaded.fire({

‎extensions/ql-vscode/src/stories/variant-analysis/RepoRow.stories.tsx‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import { createMockRepositoryWithMetadata } from "../../../test/factories/remote
1515

1616
import * as analysesResults from "../remote-queries/data/analysesResultsMessage.json";
1717
import * as rawResults from "../remote-queries/data/rawResults.json";
18-
import { RepoRow } from "../../view/variant-analysis/RepoRow";
18+
import { RepoRow, RepoRowProps } from "../../view/variant-analysis/RepoRow";
1919

2020
export default {
2121
title: "Variant Analysis/Repo Row",
@@ -29,7 +29,7 @@ export default {
2929
],
3030
} as ComponentMeta<typeof RepoRow>;
3131

32-
const Template: ComponentStory<typeof RepoRow> = (args) => (
32+
const Template: ComponentStory<typeof RepoRow> = (args: RepoRowProps) => (
3333
<RepoRow {...args} />
3434
);
3535

@@ -77,23 +77,44 @@ SucceededDownloading.args = {
7777
...Pending.args,
7878
status: VariantAnalysisRepoStatus.Succeeded,
7979
resultCount: 198,
80-
downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
80+
downloadState: {
81+
repositoryId: 63537249,
82+
downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
83+
},
84+
};
85+
86+
export const SucceededDownloadingWithPercentage = Template.bind({});
87+
SucceededDownloadingWithPercentage.args = {
88+
...Pending.args,
89+
status: VariantAnalysisRepoStatus.Succeeded,
90+
resultCount: 198,
91+
downloadState: {
92+
repositoryId: 63537249,
93+
downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
94+
downloadPercentage: 42,
95+
},
8196
};
8297

8398
export const SucceededSuccessfulDownload = Template.bind({});
8499
SucceededSuccessfulDownload.args = {
85100
...Pending.args,
86101
status: VariantAnalysisRepoStatus.Succeeded,
87102
resultCount: 198,
88-
downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Succeeded,
103+
downloadState: {
104+
repositoryId: 63537249,
105+
downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Succeeded,
106+
},
89107
};
90108

91109
export const SucceededFailedDownload = Template.bind({});
92110
SucceededFailedDownload.args = {
93111
...Pending.args,
94112
status: VariantAnalysisRepoStatus.Succeeded,
95113
resultCount: 198,
96-
downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Failed,
114+
downloadState: {
115+
repositoryId: 63537249,
116+
downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Failed,
117+
},
97118
};
98119

99120
export const InterpretedResults = Template.bind({});

‎extensions/ql-vscode/src/view/common/icon/Codicon.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ export const Codicon = ({ name, label, className, slot }: Props) => (
1717
<CodiconIcon
1818
role="img"
1919
aria-label={label}
20+
title={label}
2021
className={classNames("codicon", `codicon-${name}`, className)}
2122
slot={slot}
2223
/>

‎extensions/ql-vscode/src/view/variant-analysis/RepoRow.tsx‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
isCompletedAnalysisRepoStatus,
77
VariantAnalysisRepoStatus,
88
VariantAnalysisScannedRepositoryDownloadStatus,
9+
VariantAnalysisScannedRepositoryState,
910
} from "../../remote-queries/shared/variant-analysis";
1011
import { formatDecimal } from "../../pure/number";
1112
import {
@@ -91,7 +92,7 @@ export type RepoRowProps = {
9192
repository: Partial<RepositoryWithMetadata> &
9293
Pick<RepositoryWithMetadata, "fullName">;
9394
status?: VariantAnalysisRepoStatus;
94-
downloadStatus?: VariantAnalysisScannedRepositoryDownloadStatus;
95+
downloadState?: VariantAnalysisScannedRepositoryState;
9596
resultCount?: number;
9697

9798
interpretedResults?: AnalysisAlert[];
@@ -163,7 +164,7 @@ const filterRepoRowExpandedTelemetry = (v: boolean) => v;
163164
export const RepoRow = ({
164165
repository,
165166
status,
166-
downloadStatus,
167+
downloadState,
167168
resultCount,
168169
interpretedResults,
169170
rawResults,
@@ -185,7 +186,7 @@ export const RepoRow = ({
185186
if (
186187
resultsLoaded ||
187188
status !== VariantAnalysisRepoStatus.Succeeded ||
188-
downloadStatus !==
189+
downloadState?.downloadStatus !==
189190
VariantAnalysisScannedRepositoryDownloadStatus.Succeeded
190191
) {
191192
setExpanded((oldIsExpanded) => !oldIsExpanded);
@@ -203,7 +204,7 @@ export const RepoRow = ({
203204
resultsLoaded,
204205
repository.fullName,
205206
status,
206-
downloadStatus,
207+
downloadState,
207208
setExpanded,
208209
]);
209210

@@ -234,10 +235,11 @@ export const RepoRow = ({
234235
[onSelectedChange, repository],
235236
);
236237

237-
const disabled = !canExpand(status, downloadStatus) || resultsLoading;
238+
const disabled =
239+
!canExpand(status, downloadState?.downloadStatus) || resultsLoading;
238240
const expandableContentLoaded = isExpandableContentLoaded(
239241
status,
240-
downloadStatus,
242+
downloadState?.downloadStatus,
241243
resultsLoaded,
242244
);
243245

@@ -252,7 +254,9 @@ export const RepoRow = ({
252254
onChange={onChangeCheckbox}
253255
onClick={onClickCheckbox}
254256
checked={selected}
255-
disabled={!repository.id || !canSelect(status, downloadStatus)}
257+
disabled={
258+
!repository.id || !canSelect(status, downloadState?.downloadStatus)
259+
}
256260
/>
257261
{isExpanded && (
258262
<ExpandCollapseCodicon name="chevron-down" label="Collapse" />
@@ -278,11 +282,17 @@ export const RepoRow = ({
278282
)}
279283
{!status && <WarningIcon />}
280284
</span>
281-
{downloadStatus ===
285+
{downloadState?.downloadStatus ===
282286
VariantAnalysisScannedRepositoryDownloadStatus.InProgress && (
283-
<LoadingIcon label="Downloading" />
287+
<LoadingIcon
288+
label={
289+
downloadState.downloadPercentage !== undefined
290+
? `Downloading: ${downloadState.downloadPercentage}%`
291+
: "Downloading"
292+
}
293+
/>
284294
)}
285-
{downloadStatus ===
295+
{downloadState?.downloadStatus ===
286296
VariantAnalysisScannedRepositoryDownloadStatus.Failed && (
287297
<WarningIcon label="Failed to download the results" />
288298
)}
@@ -296,7 +306,7 @@ export const RepoRow = ({
296306
{isExpanded && expandableContentLoaded && (
297307
<AnalyzedRepoItemContent
298308
status={status}
299-
downloadStatus={downloadStatus}
309+
downloadStatus={downloadState?.downloadStatus}
300310
interpretedResults={interpretedResults}
301311
rawResults={rawResults}
302312
/>

‎extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisAnalyzedRepos.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ export const VariantAnalysisAnalyzedRepos = ({
8989
key={repository.repository.id}
9090
repository={repository.repository}
9191
status={repository.analysisStatus}
92-
downloadStatus={state?.downloadStatus}
92+
downloadState={state}
9393
resultCount={repository.resultCount}
9494
interpretedResults={results?.interpretedResults}
9595
rawResults={results?.rawResults}

0 commit comments

Comments
 (0)