diff --git a/platform/wab/src/wab/server/loader/gen-code-bundle.spec.ts b/platform/wab/src/wab/server/loader/gen-code-bundle.spec.ts index 96f94d026a9..e1ff43a6c83 100644 --- a/platform/wab/src/wab/server/loader/gen-code-bundle.spec.ts +++ b/platform/wab/src/wab/server/loader/gen-code-bundle.spec.ts @@ -4,6 +4,58 @@ import { LOADER_CODEGEN_OPTS_DEFAULTS, } from "@/wab/server/loader/gen-code-bundle"; +const BASE_OPTS = { + platformOptions: {}, + i18nTagPrefix: undefined, +} as const; + +describe("makeExportOpts", () => { + it("defaults platform to react when not specified", () => { + const opts = _testonly.makeExportOpts({ ...BASE_OPTS, loaderVersion: 1 }); + expect(opts.platform).toBe("react"); + }); + + it("sets loaderVersion feature flags correctly", () => { + const v1 = _testonly.makeExportOpts({ ...BASE_OPTS, loaderVersion: 1 }); + expect(v1.defaultExportHostLessComponents).toBe(true); + expect(v1.useComponentSubstitutionApi).toBe(false); + expect(v1.useGlobalVariantsSubstitutionApi).toBe(false); + expect(v1.useCodeComponentHelpersRegistry).toBe(false); + + const v10 = _testonly.makeExportOpts({ ...BASE_OPTS, loaderVersion: 10 }); + expect(v10.defaultExportHostLessComponents).toBe(false); + expect(v10.useComponentSubstitutionApi).toBe(true); + expect(v10.useGlobalVariantsSubstitutionApi).toBe(true); + expect(v10.useCodeComponentHelpersRegistry).toBe(true); + }); + + it("includes localization when i18nKeyScheme is provided", () => { + const opts = _testonly.makeExportOpts({ + ...BASE_OPTS, + loaderVersion: 1, + i18nKeyScheme: "hash", + i18nTagPrefix: "x-", + }); + expect(opts.localization).toEqual({ keyScheme: "hash", tagPrefix: "x-" }); + }); + + it("omits localization when i18nKeyScheme is not provided", () => { + const opts = _testonly.makeExportOpts({ ...BASE_OPTS, loaderVersion: 1 }); + expect(opts.localization).toBeUndefined(); + }); + + it("produces a key consistent with LOADER_CODEGEN_OPTS_DEFAULTS for default inputs", () => { + const opts = _testonly.makeExportOpts({ ...BASE_OPTS, loaderVersion: 1 }); + expect(opts).toMatchObject({ + ...LOADER_CODEGEN_OPTS_DEFAULTS, + defaultExportHostLessComponents: true, + useComponentSubstitutionApi: false, + useGlobalVariantsSubstitutionApi: false, + useCodeComponentHelpersRegistry: false, + }); + }); +}); + describe("makeBundleBucketPath/extractBundleKeyProjectIds", () => { it("should work", () => { const bundleKey = _testonly.makeBundleBucketPath({ diff --git a/platform/wab/src/wab/server/loader/gen-code-bundle.ts b/platform/wab/src/wab/server/loader/gen-code-bundle.ts index 8c550dedc8a..4486df6f2ff 100644 --- a/platform/wab/src/wab/server/loader/gen-code-bundle.ts +++ b/platform/wab/src/wab/server/loader/gen-code-bundle.ts @@ -6,7 +6,10 @@ import { VersionToSync, } from "@/wab/server/loader/resolve-projects"; import { withSpan } from "@/wab/server/util/apm-util"; -import { upsertS3CacheEntry } from "@/wab/server/util/s3-util"; +import { + tryGetS3CacheEntry, + upsertS3CacheEntry, +} from "@/wab/server/util/s3-util"; import { CachedCodegenOutputBundle, ComponentReference, @@ -70,6 +73,11 @@ export async function genPublishedLoaderCodeBundle( ) { const { projectVersions } = opts; + const cachedBundle = await tryGetCachedPublishedBundle(projectVersions, opts); + if (cachedBundle !== null) { + return cachedBundle; + } + const allProjectVersions = await withSpan( "loader-resolve-deps", async () => ({ @@ -163,23 +171,7 @@ async function genLoaderCodeBundleForProjectVersions( skipHead?: boolean; } ) { - const exportOpts: ExportOpts = { - ...LOADER_CODEGEN_OPTS_DEFAULTS, - platform: (opts.platform ?? - LOADER_CODEGEN_OPTS_DEFAULTS.platform) as ExportOpts["platform"], - platformOptions: opts.platformOptions, - defaultExportHostLessComponents: opts.loaderVersion > 2 ? false : true, - useComponentSubstitutionApi: opts.loaderVersion >= 6 ? true : false, - useGlobalVariantsSubstitutionApi: opts.loaderVersion >= 7 ? true : false, - useCodeComponentHelpersRegistry: opts.loaderVersion >= 10 ? true : false, - ...(opts.i18nKeyScheme && { - localization: { - keyScheme: opts.i18nKeyScheme ?? "content", - tagPrefix: opts.i18nTagPrefix, - }, - }), - skipHead: opts.skipHead, - }; + const exportOpts = makeExportOpts(opts); const codegenProject = async ( projectId: string, @@ -383,6 +375,66 @@ function makeExportOptsKey(opts: ExportOpts) { return createHash("sha256").update(str).digest("hex"); } +function makeExportOpts(opts: { + platform?: string; + platformOptions: ExportPlatformOptions; + loaderVersion: number; + i18nKeyScheme?: LocalizationKeyScheme; + i18nTagPrefix: string | undefined; + skipHead?: boolean; +}): ExportOpts { + return { + ...LOADER_CODEGEN_OPTS_DEFAULTS, + platform: (opts.platform ?? + LOADER_CODEGEN_OPTS_DEFAULTS.platform) as ExportOpts["platform"], + platformOptions: opts.platformOptions, + defaultExportHostLessComponents: opts.loaderVersion > 2 ? false : true, + useComponentSubstitutionApi: opts.loaderVersion >= 6 ? true : false, + useGlobalVariantsSubstitutionApi: opts.loaderVersion >= 7 ? true : false, + useCodeComponentHelpersRegistry: opts.loaderVersion >= 10 ? true : false, + ...(opts.i18nKeyScheme && { + localization: { + keyScheme: opts.i18nKeyScheme ?? "content", + tagPrefix: opts.i18nTagPrefix, + }, + }), + skipHead: opts.skipHead, + }; +} + +async function tryGetCachedPublishedBundle( + projectVersions: Record, + opts: { + platform?: string; + platformOptions: ExportPlatformOptions; + loaderVersion: number; + browserOnly: boolean; + i18nKeyScheme?: LocalizationKeyScheme; + i18nTagPrefix: string | undefined; + skipHead?: boolean; + } +): Promise> | null> { + const exportOpts = makeExportOpts(opts); + const bundleKey = makeBundleBucketPath({ + projectVersions, + platform: exportOpts.platform, + loaderVersion: opts.loaderVersion, + browserOnly: opts.browserOnly, + exportOpts, + }); + const cached = await tryGetS3CacheEntry({ + bucket: LOADER_ASSETS_BUCKET, + key: bundleKey, + deserialize: (str) => JSON.parse(str), + }); + if (cached !== null) { + cached.bundleKey = bundleKey; + return cached; + } + return null; +} + export const _testonly = { makeBundleBucketPath, + makeExportOpts, }; diff --git a/platform/wab/src/wab/server/util/s3-util.spec.ts b/platform/wab/src/wab/server/util/s3-util.spec.ts new file mode 100644 index 00000000000..8a142197566 --- /dev/null +++ b/platform/wab/src/wab/server/util/s3-util.spec.ts @@ -0,0 +1,122 @@ +import { _testonly, tryGetS3CacheEntry, upsertS3CacheEntry } from "@/wab/server/util/s3-util"; + +const mockGetObjectPromise = jest.fn(); +const mockPutObjectPromise = jest.fn(); +const mockS3Instance = { + getObject: jest.fn(), + putObject: jest.fn(), +}; + +jest.mock("aws-sdk/clients/s3", () => jest.fn()); + +beforeEach(() => { + // resetMocks: true clears implementations between tests — re-apply each time + const S3 = require("aws-sdk/clients/s3"); + S3.mockImplementation(() => mockS3Instance); + mockS3Instance.getObject.mockReturnValue({ promise: mockGetObjectPromise }); + mockS3Instance.putObject.mockReturnValue({ promise: mockPutObjectPromise }); + _testonly.resetS3Client(); +}); + +describe("tryGetS3CacheEntry", () => { + it("returns deserialized value on cache hit", async () => { + mockGetObjectPromise.mockResolvedValue({ + Body: Buffer.from(JSON.stringify({ ok: true })), + }); + const result = await tryGetS3CacheEntry({ + bucket: "b", + key: "k", + deserialize: JSON.parse, + }); + expect(result).toEqual({ ok: true }); + }); + + it("returns null on cache miss", async () => { + mockGetObjectPromise.mockRejectedValue({ code: "NoSuchKey" }); + const result = await tryGetS3CacheEntry({ + bucket: "b", + key: "k", + deserialize: JSON.parse, + }); + expect(result).toBeNull(); + }); + + it("returns null on other S3 errors", async () => { + mockGetObjectPromise.mockRejectedValue(new Error("AccessDenied")); + const result = await tryGetS3CacheEntry({ + bucket: "b", + key: "k", + deserialize: JSON.parse, + }); + expect(result).toBeNull(); + }); + + it("rethrows TimeoutError", async () => { + const err = Object.assign(new Error("S3 timeout"), { code: "TimeoutError" }); + mockGetObjectPromise.mockRejectedValue(err); + await expect( + tryGetS3CacheEntry({ bucket: "b", key: "k", deserialize: JSON.parse }) + ).rejects.toThrow("S3 timeout"); + }); +}); + +describe("upsertS3CacheEntry", () => { + it("returns deserialized value on cache hit without calling compute", async () => { + mockGetObjectPromise.mockResolvedValue({ + Body: Buffer.from('"cached"'), + }); + const compute = jest.fn(); + const result = await upsertS3CacheEntry({ + bucket: "b", + key: "k", + compute, + serialize: JSON.stringify, + deserialize: JSON.parse, + }); + expect(result).toBe("cached"); + expect(compute).not.toHaveBeenCalled(); + }); + + it("computes and stores value on cache miss", async () => { + mockGetObjectPromise.mockRejectedValue({ code: "NoSuchKey" }); + mockPutObjectPromise.mockResolvedValue({}); + const result = await upsertS3CacheEntry({ + bucket: "b", + key: "k", + compute: async () => "computed", + serialize: JSON.stringify, + deserialize: JSON.parse, + }); + expect(result).toBe("computed"); + expect(mockPutObjectPromise).toHaveBeenCalled(); + }); + + it("rethrows TimeoutError", async () => { + const err = Object.assign(new Error("S3 timeout"), { code: "TimeoutError" }); + mockGetObjectPromise.mockRejectedValue(err); + await expect( + upsertS3CacheEntry({ + bucket: "b", + key: "k", + compute: async () => "x", + serialize: JSON.stringify, + deserialize: JSON.parse, + }) + ).rejects.toThrow("S3 timeout"); + }); +}); + +describe("_testonly.resetS3Client", () => { + it("forces a new S3 instance to be created on next call", async () => { + const S3 = require("aws-sdk/clients/s3"); + mockGetObjectPromise.mockResolvedValue({ Body: Buffer.from('"a"') }); + + await tryGetS3CacheEntry({ bucket: "b", key: "k", deserialize: JSON.parse }); + const beforeReset = S3.mock.instances.length; + + _testonly.resetS3Client(); + await tryGetS3CacheEntry({ bucket: "b", key: "k", deserialize: JSON.parse }); + + expect(S3.mock.instances.length).toBe(beforeReset + 1); + }); +}); diff --git a/platform/wab/src/wab/server/util/s3-util.ts b/platform/wab/src/wab/server/util/s3-util.ts index 2bfe1e4e788..7f2e4524433 100644 --- a/platform/wab/src/wab/server/util/s3-util.ts +++ b/platform/wab/src/wab/server/util/s3-util.ts @@ -3,6 +3,31 @@ import { ensureInstance } from "@/wab/shared/common"; import S3 from "aws-sdk/clients/s3"; import path from "path"; +let _s3: S3 | undefined; +function getS3Client(): S3 { + return (_s3 ??= new S3({ endpoint: process.env.S3_ENDPOINT })); +} + +export async function tryGetS3CacheEntry(opts: { + bucket: string; + key: string; + deserialize: (str: string) => T; +}): Promise { + const { bucket, key, deserialize } = opts; + const s3 = getS3Client(); + try { + const obj = await s3.getObject({ Bucket: bucket, Key: key }).promise(); + const serialized = ensureInstance(obj.Body, Buffer).toString("utf8"); + logger().info(`S3 cache hit for ${bucket} ${key}`); + return deserialize(serialized); + } catch (err) { + if (err.code === "TimeoutError") { + throw err; + } + return null; + } +} + export async function upsertS3CacheEntry(opts: { bucket: string; key: string; @@ -11,7 +36,7 @@ export async function upsertS3CacheEntry(opts: { deserialize: (str: string) => T; }) { const { bucket, key, compute: f, serialize, deserialize } = opts; - const s3 = new S3({ endpoint: process.env.S3_ENDPOINT }); + const s3 = getS3Client(); try { const obj = await s3 @@ -49,13 +74,19 @@ export async function upsertS3CacheEntry(opts: { } } +export const _testonly = { + resetS3Client: () => { + _s3 = undefined; + }, +}; + export async function uploadFilesToS3(opts: { bucket: string; key: string; files: Record; }) { const { bucket, key, files } = opts; - const s3 = new S3({ endpoint: process.env.S3_ENDPOINT }); + const s3 = getS3Client(); await Promise.all( Object.entries(files).map(async ([file, content]) => { await s3