Inject the Mockttp CT log into Android trusted CT log config

Author: pimterry

package-lock.json

@@ -74,6 +74,7 @@
     "env-paths": "^1.0.0",
     "event-stream": "^4.0.1",
     "express": "^5.2.1",
+    "flatbuffers": "25.1.24",
     "frida-js": "^0.4.0",
     "get-port": "^7.2.0",
     "graphql": "^16.13.2",

package.json

@@ -6,5 +6,6 @@ export interface HtkConfig {
         certPath: string;
         certContent: string;
         keyLength: number;
+        certificateTransparency: boolean;
     }
 }

src/config.d.ts

@@ -66,7 +66,8 @@ async function generateHTTPSConfig(configPath: string) {
         keyPath,
         certPath,
         certContent,
-        keyLength: 2048 // Reasonably secure keys please
+        keyLength: 2048, // Reasonably secure keys please
+        certificateTransparency: true
     };
 }
 

src/index.ts

@@ -8,10 +8,16 @@ import { logError } from '../../error-tracking';
 import { waitUntil } from '../../util/promise';
 import { getCertificateFingerprint, parseCert } from '../../certificates';
 import { streamToBuffer } from '../../util/stream';
-
 export const ANDROID_TEMP = '/data/local/tmp';
 export const SYSTEM_CA_PATH = '/system/etc/security/cacerts';
 
+const CT_LOG_DIR = '/data/misc/keychain/ct';
+const CT_LOG_LIST_PATHS = {
+    v1: `${CT_LOG_DIR}/v1/current/log_list.json`,
+    v2: `${CT_LOG_DIR}/v2/current/log_list.json`,
+    v3: `${CT_LOG_DIR}/v3/current/log_list.ctfb` // Flatbuffer
+};
+
 export const EMULATOR_HOST_IPS = [
     '10.0.2.2', // Standard emulator localhost ip
     '10.0.3.2', // Genymotion localhost ip
@@ -189,11 +195,11 @@ const filterDeviceNameCache = (connectedIds: string[]) => {
 };
 
 export function stringAsStream(input: string) {
-    const contentStream = new stream.Readable();
-    contentStream._read = () => {};
-    contentStream.push(input);
-    contentStream.push(null);
-    return contentStream;
+    return bufferAsStream(Buffer.from(input, 'utf8'));
+}
+
+function bufferAsStream(input: Buffer) {
+    return stream.Readable.from([input], { objectMode: false });
 }
 
 async function run(
@@ -279,7 +285,7 @@ const runAsRootCommands = [
     (...cmd: string[]) => ['su', 'root', cmd.join(' ')]
 ];
 
-type RootCmd = (...cmd: string[]) => string[];
+export type RootCmd = (...cmd: string[]) => string[];
 
 export async function getRootCommand(adbClient: Adb.DeviceClient): Promise<RootCmd | undefined> {
     const rootTestScriptPath = `${ANDROID_TEMP}/htk-root-test.sh`;
@@ -412,6 +418,47 @@ const isMatchingCert = async (certStream: stream.Readable, expectedFingerprint:
     return expectedFingerprint === existingFingerprint;
 }
 
+// Push a script, run it as root, and (optionally) check that it reported success. The
+// script deletes itself on any exit path, so nothing is left behind on the device.
+async function runRootScript(
+    adbClient: Adb.DeviceClient,
+    runAsRoot: RootCmd,
+    scriptName: string,
+    script: string,
+    options: {
+        files?: Array<{ content: Buffer, path: string }>,
+        successMarker?: string,
+        timeout?: number,
+        skipLogging?: boolean
+    } = {}
+) {
+    const scriptPath = `${ANDROID_TEMP}/${scriptName}`;
+
+    await Promise.all([
+        ...(options.files ?? []).map(({ content, path }) =>
+            // Due to an Android bug, user mode is always duplicated to group & others. We set as
+            // read-only to avoid making these writable by others before we use them as root in a
+            // moment. More details: https://github.com/openstf/adbkit/issues/126
+            pushFile(adbClient, bufferAsStream(content), path, 0o444)
+        ),
+        pushFile(adbClient, stringAsStream(`
+            trap 'rm -f ${scriptPath}' EXIT
+            ${script}
+        `), scriptPath, 0o444)
+    ]);
+
+    const output = await run(adbClient, runAsRoot('sh', scriptPath), {
+        timeout: options.timeout ?? 10000,
+        skipLogging: options.skipLogging
+    });
+
+    if (options.successMarker && !output.includes(options.successMarker)) {
+        throw new Error(`${scriptName} failed`);
+    }
+
+    return output;
+}
+
 export async function injectSystemCertificate(
     adbClient: Adb.DeviceClient,
     runAsRoot: RootCmd,
@@ -529,6 +576,118 @@ export async function injectSystemCertificate(
     }
 }
 
+// Read device's current CT log list. Undefined if there is none, but throws if it can't
+// be read for some reason.
+export async function readCtLogList(
+    adbClient: Adb.DeviceClient,
+    runAsRoot: RootCmd
+): Promise<Buffer | undefined> {
+    // Multiple possible formats, most recent is used by preference. Can't pull directly as we
+    // need `su`, so we read on-device and base64 to avoid binary getting mangled in transfer.
+    // To confirm the exact result, we need to wrap our output with extra info, otherwise we
+    // can't reliably differentiate missing/error/truncated/OK.
+    const output = await runRootScript(adbClient, runAsRoot, 'htk-read-ct-logs.sh', `
+        for LIST_PATH in ${[
+            CT_LOG_LIST_PATHS.v3,
+            CT_LOG_LIST_PATHS.v2,
+            CT_LOG_LIST_PATHS.v1
+        ].join(' ')}; do
+            if [ -f "$LIST_PATH" ]; then
+                echo "HTK-CT-LIST $LIST_PATH"
+                base64 "$LIST_PATH"
+                echo "HTK-CT-READ $?"
+                exit 0
+            fi
+        done
+
+        echo "HTK-CT-LIST none"
+        echo "HTK-CT-READ 0"
+    `, { timeout: 3000, skipLogging: true });
+
+    const result = output.match(/HTK-CT-LIST (\S+)\r?\n([\s\S]*)HTK-CT-READ (\d+)/);
+    if (!result) {
+        throw new Error(`Could not read the device's CT log list: ${
+            output.trim().slice(0, 200) || 'no output'
+        }`);
+    }
+
+    const [, listPath, listContent, readResult] = result;
+
+    if (listPath === 'none') {
+        console.log('No existing CT log list found on device');
+        return undefined;
+    }
+
+    if (readResult !== '0') {
+        throw new Error(`Reading ${listPath} failed with status ${readResult}`);
+    }
+
+    console.log(`Read existing CT log list from ${listPath}`);
+    return Buffer.from(listContent.replace(/[^A-Za-z0-9+/=]/g, ''), 'base64');
+}
+
+export async function injectCtLogLists(
+    adbClient: Adb.DeviceClient,
+    runAsRoot: RootCmd,
+    logLists: { json: Buffer, ctfb: Buffer }
+) {
+    const jsonPath = `${ANDROID_TEMP}/htk-ct-log-list.json`;
+    const ctfbPath = `${ANDROID_TEMP}/htk-ct-log-list.ctfb`;
+
+    // Use tmpfs to shadow device's CT config with our extended version:
+    await runRootScript(adbClient, runAsRoot, 'htk-inject-ct-logs.sh', `
+            set -e # Fail on error
+
+            echo "\n---\nInjecting CT log lists:"
+
+            # Drop any previous injection, so that repeated runs don't stack up mounts. This
+            # has to be lazy: apps keep the log list mmapped, so a normal umount fails while
+            # any of them are still running.
+            for i in 1 2 3 4 5; do umount -l ${CT_LOG_DIR} 2>/dev/null || break; done
+
+            # A fresh tmpfs is labelled u:object_r:tmpfs:s0, which apps can't read, so we
+            # relabel everything to match the SELinux context Android uses here normally:
+            set -- $(ls -Zd /data/misc/keychain 2>/dev/null || true)
+            CT_CONTEXT=$1
+            case "$CT_CONTEXT" in
+                u:object_r:*) ;;
+                *) CT_CONTEXT=u:object_r:keychain_data_file:s0 ;;
+            esac
+
+            mkdir -p ${CT_LOG_DIR}
+            mount -t tmpfs tmpfs ${CT_LOG_DIR}
+
+            mkdir -p ${[
+                CT_LOG_LIST_PATHS.v1,
+                CT_LOG_LIST_PATHS.v2,
+                CT_LOG_LIST_PATHS.v3
+            ].map((listPath) => path.posix.dirname(listPath)).join(' ')}
+
+            mv ${jsonPath} ${CT_LOG_LIST_PATHS.v1}
+            cp ${CT_LOG_LIST_PATHS.v1} ${CT_LOG_LIST_PATHS.v2}
+            mv ${ctfbPath} ${CT_LOG_LIST_PATHS.v3}
+
+            # Make everything as readable as the real log lists are:
+            chown -R root:root ${CT_LOG_DIR}
+            chmod -R 755 ${CT_LOG_DIR}
+            chmod 644 ${Object.values(CT_LOG_LIST_PATHS).join(' ')}
+            chcon -R $CT_CONTEXT ${CT_LOG_DIR}
+
+            # Read-only, so that the daily CT log list update can't replace our lists. The
+            # update fails & is retried later instead, which is harmless.
+            mount -o remount,ro tmpfs ${CT_LOG_DIR} ||
+                echo 'Could not remount the CT log lists as read-only'
+
+            echo "CT log lists successfully injected\n---\n"
+    `, {
+        files: [
+            { content: logLists.json, path: jsonPath },
+            { content: logLists.ctfb, path: ctfbPath }
+        ],
+        successMarker: 'CT log lists successfully injected'
+    });
+}
+
 export async function setChromeFlags(
     adbClient: Adb.DeviceClient,
     runAsRoot: RootCmd,

src/interceptors/android/adb-commands.ts

@@ -21,8 +21,17 @@ import {
     startActivity,
     createPersistentReverseTunnel,
     closeReverseTunnel,
-    EMULATOR_HOST_IPS
+    readCtLogList,
+    injectCtLogLists,
+    EMULATOR_HOST_IPS,
+    RootCmd
 } from './adb-commands';
+import {
+    buildCtLogList,
+    ctLogListIncludesCa,
+    parseCtLogList,
+    serializeCtLogLists
+} from './ct-log-list';
 import { streamLatestApk, clearAllApks } from './fetch-apk';
 import { parseCert, getCertificateFingerprint, getCertificateSubjectHash } from '../../certificates';
 import { getReachableInterfaces } from '../../util/network';
@@ -72,7 +81,7 @@ export class AndroidAdbInterceptor implements Interceptor {
     }): Promise<void | {}> {
         const deviceClient = new DeviceClient(this.adbClient, options.deviceId);
 
-        await this.injectSystemCertIfPossible(deviceClient, this.config.https.certContent);
+        await this.setupCertificateTrust(deviceClient, this.config.https.certContent);
 
         if (!(await deviceClient.isInstalled('tech.httptoolkit.android.v1'))) {
             console.log("App not installed, installing...");
@@ -160,13 +169,19 @@ export class AndroidAdbInterceptor implements Interceptor {
         );
     }
 
-    private async injectSystemCertIfPossible(deviceClient: DeviceClient, certContent: string) {
+    private async setupCertificateTrust(deviceClient: DeviceClient, certContent: string) {
         const rootCmd = await getRootCommand(deviceClient);
         if (!rootCmd) {
             console.log('Root not available, skipping cert injection');
             return;
         }
 
+        // Read the device's CT log list in parallel with cert setup:
+        const existingCtLogList = this.config.https.certificateTransparency
+            ? readCtLogList(deviceClient, rootCmd)
+            : undefined;
+        existingCtLogList?.catch(() => {}); // Errors are handled where we await this below
+
         const cert = parseCert(certContent);
 
         try {
@@ -191,10 +206,15 @@ export class AndroidAdbInterceptor implements Interceptor {
                 console.log("Cert already installed, nothing to do");
             }
 
+            if (existingCtLogList) {
+                await this.injectCtLogListIfNeeded(deviceClient, rootCmd, certContent, existingCtLogList)
+                    .catch(logError); // Continue but log the failure
+            }
+
             const spkiFingerprint = await generateSPKIFingerprint(certContent);
 
-            // Chrome requires system certificates to use certificate transparency, which we can't do. To work
-            // around this, we need to explicitly trust our certificate in Chrome:
+            // Chrome only trusts SCTs from the CT logs it recognizes itself, which we can't
+            // provide. To work around that, we explicitly trust our certificate in Chrome:
             await setChromeFlags(deviceClient, rootCmd, [
                 `--ignore-certificate-errors-spki-list=${spkiFingerprint}`
             ]);
@@ -203,4 +223,27 @@ export class AndroidAdbInterceptor implements Interceptor {
             logError(e);
         }
     }
-}
+
+    // From Android 16 (and by default from Android 17) apps can require certificate
+    // transparency, which rejects any certificate without SCTs from known CT logs - including
+    // ours. Mockttp stamps our certificates with SCTs from logs derived from the CA, so here
+    // we add those logs to the device's log list to make those SCTs verifiable.
+    private async injectCtLogListIfNeeded(
+        deviceClient: DeviceClient,
+        rootCmd: RootCmd,
+        certContent: string,
+        existingListRead: Promise<Buffer | undefined>
+    ) {
+        const existingList = parseCtLogList(await existingListRead);
+
+        if (ctLogListIncludesCa(existingList, certContent)) {
+            console.log("CT log list already installed, nothing to do");
+            return;
+        }
+
+        await injectCtLogLists(deviceClient, rootCmd,
+            serializeCtLogLists(buildCtLogList(certContent, existingList))
+        );
+        console.log('CT log list injected');
+    }
+}