From fed4c2db2ce1db934f5e5c865a1896e9a167fe7d Mon Sep 17 00:00:00 2001 From: Connor OMalley Date: Wed, 29 Jul 2026 15:43:14 +0100 Subject: [PATCH 01/10] Matching Prod instance types for CODE stage --- dotcom-rendering/cdk/bin/cdk.ts | 2 +- .../cdk/lib/__snapshots__/renderingStack.test.ts.snap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dotcom-rendering/cdk/bin/cdk.ts b/dotcom-rendering/cdk/bin/cdk.ts index ed16e84ea4e..6b5b96a3188 100644 --- a/dotcom-rendering/cdk/bin/cdk.ts +++ b/dotcom-rendering/cdk/bin/cdk.ts @@ -102,7 +102,7 @@ export const TagPageRenderingPropsCODE: RenderingCDKStackProps = { stage: 'CODE', domainName: 'tag-page-rendering.code.dev-guardianapis.com', scaling: { minimumInstances: 1, maximumInstances: 3 }, - instanceType: InstanceType.of(InstanceClass.T4G, InstanceSize.SMALL), + instanceType: InstanceType.of(InstanceClass.C8G, InstanceSize.MEDIUM), imageIdentifier: process.env.IMAGE_DIGEST ?? 'DEV', }; diff --git a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap index 3ca44abe4b8..7bbbf1b24ca 100644 --- a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap +++ b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap @@ -3345,7 +3345,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE "ImageId": { "Ref": "AMITagpagerendering", }, - "InstanceType": "t4g.small", + "InstanceType": "c8g.medium", "MetadataOptions": { "HttpTokens": "required", "InstanceMetadataTags": "enabled", From b726dc2124af13715406dde15c7708a2963cb09f Mon Sep 17 00:00:00 2001 From: Connor OMalley Date: Fri, 31 Jul 2026 15:04:47 +0100 Subject: [PATCH 02/10] Reduce memory and CPU for load test. --- dotcom-rendering/cdk/lib/renderingStack.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dotcom-rendering/cdk/lib/renderingStack.ts b/dotcom-rendering/cdk/lib/renderingStack.ts index 05b116e337b..5b53d409fc9 100644 --- a/dotcom-rendering/cdk/lib/renderingStack.ts +++ b/dotcom-rendering/cdk/lib/renderingStack.ts @@ -279,10 +279,8 @@ export class RenderingCDKStack extends CDKStack { ecsProps: { repositoryName: 'guardian/dotcom-rendering', imageIdentifier, - - // TODO tune these values - memoryLimitMiB: 2048, - cpu: 1024, + memoryLimitMiB: 1024, + cpu: 512, scaling: { minimumTasks: 1, maximumTasks: 2, From 3712f94b9a67a7cf215bed3736c900ccf5dc8389 Mon Sep 17 00:00:00 2001 From: Connor OMalley Date: Mon, 3 Aug 2026 15:46:49 +0100 Subject: [PATCH 03/10] CPU scaling for ECS. --- .../__snapshots__/renderingStack.test.ts.snap | 25 ++++++++++++++++--- dotcom-rendering/cdk/lib/renderingStack.ts | 18 +++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap index 7bbbf1b24ca..8f43613270e 100644 --- a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap +++ b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap @@ -2056,7 +2056,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE "EcsTaskDefinitionTaskRoleB7B6D8DD", ], "Properties": { - "MaxCapacity": 2, + "MaxCapacity": 9, "MinCapacity": 1, "ResourceId": { "Fn::Join": [ @@ -2097,6 +2097,25 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE }, "Type": "AWS::ApplicationAutoScaling::ScalableTarget", }, + "EcsServiceTaskCountTargetCpuScalingA9C91D4C": { + "DependsOn": [ + "EcsTaskDefinitionTaskRoleB7B6D8DD", + ], + "Properties": { + "PolicyName": "TagPageRenderingCODEEcsServiceTaskCountTargetCpuScaling4DFA1A0F", + "PolicyType": "TargetTrackingScaling", + "ScalingTargetId": { + "Ref": "EcsServiceTaskCountTarget02FCCE22", + }, + "TargetTrackingScalingPolicyConfiguration": { + "PredefinedMetricSpecification": { + "PredefinedMetricType": "ECSServiceAverageCPUUtilization", + }, + "TargetValue": 50, + }, + }, + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy", + }, "EcsTaskDefinition63157ED3": { "Properties": { "ContainerDefinitions": [ @@ -2231,7 +2250,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE "VersionConsistency": "disabled", }, ], - "Cpu": "1024", + "Cpu": "512", "ExecutionRoleArn": { "Fn::GetAtt": [ "EcsTaskDefinitionExecutionRoleBE450C73", @@ -2239,7 +2258,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE ], }, "Family": "TagPageRenderingCODEEcsTaskDefinition616FF027", - "Memory": "2048", + "Memory": "1024", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", diff --git a/dotcom-rendering/cdk/lib/renderingStack.ts b/dotcom-rendering/cdk/lib/renderingStack.ts index 5b53d409fc9..3f56bf140ae 100644 --- a/dotcom-rendering/cdk/lib/renderingStack.ts +++ b/dotcom-rendering/cdk/lib/renderingStack.ts @@ -17,7 +17,7 @@ import { Metric, Unit } from 'aws-cdk-lib/aws-cloudwatch'; import { SnsAction } from 'aws-cdk-lib/aws-cloudwatch-actions'; import type { InstanceType } from 'aws-cdk-lib/aws-ec2'; import { Peer } from 'aws-cdk-lib/aws-ec2'; -import type { CfnService } from 'aws-cdk-lib/aws-ecs'; +import type { CfnService, ScalableTaskCount } from 'aws-cdk-lib/aws-ecs'; import { ClusterSettings } from 'aws-cdk-lib/aws-ecs/mixins'; import { Subscription, SubscriptionProtocol, Topic } from 'aws-cdk-lib/aws-sns'; import { StringParameter } from 'aws-cdk-lib/aws-ssm'; @@ -283,7 +283,7 @@ export class RenderingCDKStack extends CDKStack { cpu: 512, scaling: { minimumTasks: 1, - maximumTasks: 2, + maximumTasks: 9, }, }, @@ -331,6 +331,20 @@ export class RenderingCDKStack extends CDKStack { }, ], }); + + // Until we make these changes in GuCDK, we have to be a bit hacky to set the CPU Scaling option. + const ecsScalableTarget = app.ecsService.node.tryFindChild( + 'TaskCount', + ) as ScalableTaskCount; + if (ecsScalableTarget) { + ecsScalableTarget.scaleOnCpuUtilization('CpuScaling', { + targetUtilizationPercent: 50, + }); + } else { + throw new Error( + 'Could not create CPU scaling policy for ECS', + ); + } } } From 3a0eb1ab987f8b7e6f3979bbd11068eaf2aa33f5 Mon Sep 17 00:00:00 2001 From: Connor OMalley Date: Tue, 4 Aug 2026 16:01:38 +0100 Subject: [PATCH 04/10] Action logs for ECS --- .../__snapshots__/renderingStack.test.ts.snap | 139 ++++++++++++++++++ dotcom-rendering/cdk/lib/renderingStack.ts | 62 +++++++- 2 files changed, 200 insertions(+), 1 deletion(-) diff --git a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap index 8f43613270e..93141526a88 100644 --- a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap +++ b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap @@ -1956,6 +1956,145 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE }, "Type": "AWS::IAM::Policy", }, + "EcsActionDelivery": { + "Properties": { + "DeliveryDestinationArn": { + "Fn::GetAtt": [ + "EcsActionDeliveryDestination", + "Arn", + ], + }, + "DeliverySourceName": "tag-page-rendering-CODE-source", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::Delivery", + }, + "EcsActionDeliveryDestination": { + "Properties": { + "DestinationResourceArn": { + "Fn::GetAtt": [ + "EcsActionLogGroup559EC54A", + "Arn", + ], + }, + "Name": "tag-page-rendering-CODE-destination", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::DeliveryDestination", + }, + "EcsActionDeliverySource": { + "Properties": { + "LogType": "ACTION_LOGS", + "Name": "tag-page-rendering-CODE-source", + "ResourceArn": { + "Fn::GetAtt": [ + "tagpagerenderingEcsClusterE7696595", + "Arn", + ], + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::DeliverySource", + }, + "EcsActionLogGroup559EC54A": { + "DeletionPolicy": "Delete", + "Properties": { + "LogGroupName": "/aws/vendedlogs/ecs/action-logs/tag-page-rendering-CODE", + "RetentionInDays": 7, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Delete", + }, + "EcsActionLogGroupPolicyResourcePolicyD8381A26": { + "Properties": { + "PolicyDocument": { + "Fn::Join": [ + "", + [ + "{"Statement":[{"Action":["logs:CreateLogStream","logs:PutLogEvents"],"Effect":"Allow","Principal":{"Service":"delivery.logs.amazonaws.com"},"Resource":"", + { + "Fn::GetAtt": [ + "EcsActionLogGroup559EC54A", + "Arn", + ], + }, + ""}],"Version":"2012-10-17"}", + ], + ], + }, + "PolicyName": "TagPageRenderingCODEEcsActionLogGroupPolicy1FA5FF4D", + }, + "Type": "AWS::Logs::ResourcePolicy", + }, "EcsService81FC6EF6": { "DependsOn": [ "EcsTaskDefinitionTaskRoleB7B6D8DD", diff --git a/dotcom-rendering/cdk/lib/renderingStack.ts b/dotcom-rendering/cdk/lib/renderingStack.ts index 3f56bf140ae..e3a8ceaa429 100644 --- a/dotcom-rendering/cdk/lib/renderingStack.ts +++ b/dotcom-rendering/cdk/lib/renderingStack.ts @@ -10,7 +10,12 @@ import { GuCname } from '@guardian/cdk/lib/constructs/dns/dns-records'; import { GuAllowPolicy } from '@guardian/cdk/lib/constructs/iam'; import { GuLoadBalancedAppExperimental } from '@guardian/cdk/lib/experimental/patterns/gu-load-balanced-app'; import type { GuAsgCapacity } from '@guardian/cdk/lib/types'; -import { aws_cloudwatch, type App as CDKApp, Duration } from 'aws-cdk-lib'; +import { + aws_cloudwatch, + type App as CDKApp, + Duration, + RemovalPolicy, +} from 'aws-cdk-lib'; import type { ScalingInterval } from 'aws-cdk-lib/aws-applicationautoscaling'; import { AdjustmentType, StepScalingPolicy } from 'aws-cdk-lib/aws-autoscaling'; import { Metric, Unit } from 'aws-cdk-lib/aws-cloudwatch'; @@ -19,6 +24,14 @@ import type { InstanceType } from 'aws-cdk-lib/aws-ec2'; import { Peer } from 'aws-cdk-lib/aws-ec2'; import type { CfnService, ScalableTaskCount } from 'aws-cdk-lib/aws-ecs'; import { ClusterSettings } from 'aws-cdk-lib/aws-ecs/mixins'; +import { Effect, PolicyStatement, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { + CfnDelivery, + CfnDeliveryDestination, + CfnDeliverySource, + LogGroup, + RetentionDays, +} from 'aws-cdk-lib/aws-logs'; import { Subscription, SubscriptionProtocol, Topic } from 'aws-cdk-lib/aws-sns'; import { StringParameter } from 'aws-cdk-lib/aws-ssm'; import { getUserData } from './userData'; @@ -345,6 +358,53 @@ export class RenderingCDKStack extends CDKStack { 'Could not create CPU scaling policy for ECS', ); } + + // Add Action Logs + // This should provide infra level information such as how long it takes to pull an image + // + // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/action-logs-getting-started.html + const ecsServiceName = `${guApp}-${stage}`; + + const actionLogGroup = new LogGroup(this, 'EcsActionLogGroup', { + logGroupName: `/aws/vendedlogs/ecs/action-logs/${ecsServiceName}`, + retention: RetentionDays.ONE_WEEK, + removalPolicy: RemovalPolicy.DESTROY, + }); + + actionLogGroup.addToResourcePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + principals: [ + new ServicePrincipal('delivery.logs.amazonaws.com'), + ], + actions: ['logs:CreateLogStream', 'logs:PutLogEvents'], + resources: [actionLogGroup.logGroupArn], + }), + ); + + const deliverySource = new CfnDeliverySource( + this, + 'EcsActionDeliverySource', + { + name: `${ecsServiceName}-source`, + resourceArn: app.ecsService.cluster.clusterArn, + logType: 'ACTION_LOGS', + }, + ); + + const deliveryDestination = new CfnDeliveryDestination( + this, + 'EcsActionDeliveryDestination', + { + name: `${ecsServiceName}-destination`, + destinationResourceArn: actionLogGroup.logGroupArn, + }, + ); + + new CfnDelivery(this, 'EcsActionDelivery', { + deliverySourceName: deliverySource.name, + deliveryDestinationArn: deliveryDestination.attrArn, + }); } } From 32b594549b3bc0e604efa167898076667ecfa1c9 Mon Sep 17 00:00:00 2001 From: Jorge Azevedo Date: Wed, 5 Aug 2026 16:03:34 +0100 Subject: [PATCH 05/10] Add health check and event logs --- .../__snapshots__/renderingStack.test.ts.snap | 272 ++++++++++++++++++ dotcom-rendering/cdk/lib/renderingStack.ts | 144 ++++++++++ 2 files changed, 416 insertions(+) diff --git a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap index 93141526a88..6e285a43350 100644 --- a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap +++ b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap @@ -2854,6 +2854,158 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE }, "Type": "AWS::IAM::Policy", }, + "HealthCheckLogDelivery": { + "DependsOn": [ + "HealthCheckLogsDeliveryPolicy", + "HealthCheckLogSource", + ], + "Properties": { + "DeliveryDestinationArn": { + "Fn::GetAtt": [ + "HealthCheckLogDestination", + "Arn", + ], + }, + "DeliverySourceName": "tag-page-rendering-CODE-health-check", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::Delivery", + }, + "HealthCheckLogDestination": { + "Properties": { + "DestinationResourceArn": { + "Fn::GetAtt": [ + "HealthCheckLogs807FF78A", + "Arn", + ], + }, + "Name": "tag-page-rendering-CODE-health-check", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::DeliveryDestination", + }, + "HealthCheckLogSource": { + "Properties": { + "LogType": "ALB_HEALTH_CHECK_LOGS", + "Name": "tag-page-rendering-CODE-health-check", + "ResourceArn": { + "Ref": "LoadBalancerTagpagerenderingB0B7AC4E", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::DeliverySource", + }, + "HealthCheckLogs807FF78A": { + "DeletionPolicy": "Delete", + "Properties": { + "LogGroupName": "/aws/elasticloadbalancing/tag-page-rendering-CODE-health-check", + "RetentionInDays": 7, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Delete", + }, + "HealthCheckLogsDeliveryPolicy": { + "Properties": { + "PolicyDocument": { + "Fn::Join": [ + "", + [ + "{"Version":"2012-10-17","Statement":[{"Sid":"AllowLogDeliveryWrite","Effect":"Allow","Principal":{"Service":"delivery.logs.amazonaws.com"},"Action":["logs:CreateLogStream","logs:PutLogEvents"],"Resource":"arn:", + { + "Ref": "AWS::Partition", + }, + ":logs:eu-west-1:", + { + "Ref": "AWS::AccountId", + }, + ":log-group:/aws/elasticloadbalancing/tag-page-rendering-CODE-health-check:log-stream:*","Condition":{"StringEquals":{"aws:SourceAccount":"", + { + "Ref": "AWS::AccountId", + }, + ""},"ArnLike":{"aws:SourceArn":"", + { + "Fn::GetAtt": [ + "HealthCheckLogSource", + "Arn", + ], + }, + ""}}}]}", + ], + ], + }, + "PolicyName": "tag-page-rendering-CODE-health-check-logs", + }, + "Type": "AWS::Logs::ResourcePolicy", + }, "InstanceRoleTagpagerendering171BC2F9": { "Properties": { "AssumeRolePolicyDocument": { @@ -3486,6 +3638,126 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE }, "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", }, + "TaskStateChangeRule": { + "Properties": { + "EventPattern": { + "detail": { + "clusterArn": [ + { + "Fn::GetAtt": [ + "tagpagerenderingEcsClusterE7696595", + "Arn", + ], + }, + ], + }, + "detail-type": [ + "ECS Task State Change", + ], + "source": [ + "aws.ecs", + ], + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + "Targets": [ + { + "Arn": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":logs:eu-west-1:", + { + "Ref": "AWS::AccountId", + }, + ":log-group:", + { + "Ref": "TaskStateEvents4E5081CA", + }, + ], + ], + }, + "Id": "TaskStateEventsLog", + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "TaskStateEvents4E5081CA": { + "DeletionPolicy": "Delete", + "Properties": { + "LogGroupName": "/aws/events/tag-page-rendering-CODE-task-state", + "RetentionInDays": 7, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/dotcom-rendering", + }, + { + "Key": "Stack", + "Value": "frontend", + }, + { + "Key": "Stage", + "Value": "CODE", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Delete", + }, + "TaskStateEventsPolicy": { + "Properties": { + "PolicyDocument": { + "Fn::Join": [ + "", + [ + "{"Version":"2012-10-17","Statement":[{"Sid":"AllowEventBridgeToLog","Effect":"Allow","Principal":{"Service":"events.amazonaws.com"},"Action":["logs:CreateLogStream","logs:PutLogEvents"],"Resource":"", + { + "Fn::GetAtt": [ + "TaskStateEvents4E5081CA", + "Arn", + ], + }, + "","Condition":{"ArnEquals":{"aws:SourceArn":"", + { + "Fn::GetAtt": [ + "TaskStateChangeRule", + "Arn", + ], + }, + ""}}}]}", + ], + ], + }, + "PolicyName": "tag-page-rendering-CODE-task-state-events", + }, + "Type": "AWS::Logs::ResourcePolicy", + }, "frontendCODEtagpagerendering3C218821": { "DependsOn": [ "InstanceRoleTagpagerendering171BC2F9", diff --git a/dotcom-rendering/cdk/lib/renderingStack.ts b/dotcom-rendering/cdk/lib/renderingStack.ts index e3a8ceaa429..126a5d06fda 100644 --- a/dotcom-rendering/cdk/lib/renderingStack.ts +++ b/dotcom-rendering/cdk/lib/renderingStack.ts @@ -11,6 +11,7 @@ import { GuAllowPolicy } from '@guardian/cdk/lib/constructs/iam'; import { GuLoadBalancedAppExperimental } from '@guardian/cdk/lib/experimental/patterns/gu-load-balanced-app'; import type { GuAsgCapacity } from '@guardian/cdk/lib/types'; import { + ArnFormat, aws_cloudwatch, type App as CDKApp, Duration, @@ -24,11 +25,13 @@ import type { InstanceType } from 'aws-cdk-lib/aws-ec2'; import { Peer } from 'aws-cdk-lib/aws-ec2'; import type { CfnService, ScalableTaskCount } from 'aws-cdk-lib/aws-ecs'; import { ClusterSettings } from 'aws-cdk-lib/aws-ecs/mixins'; +import { CfnRule } from 'aws-cdk-lib/aws-events'; import { Effect, PolicyStatement, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { CfnDelivery, CfnDeliveryDestination, CfnDeliverySource, + CfnResourcePolicy, LogGroup, RetentionDays, } from 'aws-cdk-lib/aws-logs'; @@ -405,6 +408,147 @@ export class RenderingCDKStack extends CDKStack { deliverySourceName: deliverySource.name, deliveryDestinationArn: deliveryDestination.attrArn, }); + + // Deliver per-health-check-attempt results (PASS/FAIL, latency, target + // IP:port, reason code) straight to CloudWatch Logs as vended logs, so we + // can query them alongside task-state events to reconstruct the + // registration → healthy phase of a task's lifecycle. + // See https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-cloudwatch-logs.html + const healthCheckLogsName = `/aws/elasticloadbalancing/${ecsServiceName}-health-check`; + const healthCheckLogs = new LogGroup(this, 'HealthCheckLogs', { + logGroupName: healthCheckLogsName, + retention: RetentionDays.ONE_WEEK, + removalPolicy: RemovalPolicy.DESTROY, + }); + + const healthCheckLogSource = new CfnDeliverySource( + this, + 'HealthCheckLogSource', + { + name: `${ecsServiceName}-health-check`, + logType: 'ALB_HEALTH_CHECK_LOGS', + resourceArn: app.loadBalancer.loadBalancerArn, + }, + ); + + const healthCheckLogDestination = new CfnDeliveryDestination( + this, + 'HealthCheckLogDestination', + { + name: `${ecsServiceName}-health-check`, + destinationResourceArn: healthCheckLogs.logGroupArn, + }, + ); + + // Allow the CloudWatch Logs delivery service to write to the log group. + const healthCheckLogsPolicy = new CfnResourcePolicy( + this, + 'HealthCheckLogsDeliveryPolicy', + { + policyName: `${ecsServiceName}-health-check-logs`, + policyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'AllowLogDeliveryWrite', + Effect: 'Allow', + Principal: { + Service: 'delivery.logs.amazonaws.com', + }, + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Resource: this.formatArn({ + service: 'logs', + resource: 'log-group', + resourceName: `${healthCheckLogsName}:log-stream:*`, + arnFormat: + ArnFormat.COLON_RESOURCE_NAME, + }), + Condition: { + StringEquals: { + 'aws:SourceAccount': this.account, + }, + ArnLike: { + 'aws:SourceArn': + healthCheckLogSource.attrArn, + }, + }, + }, + ], + }), + }, + ); + + const healthCheckLogDelivery = new CfnDelivery( + this, + 'HealthCheckLogDelivery', + { + deliverySourceName: healthCheckLogSource.name, + deliveryDestinationArn: + healthCheckLogDestination.attrArn, + }, + ); + healthCheckLogDelivery.addDependency(healthCheckLogSource); + healthCheckLogDelivery.addDependency(healthCheckLogsPolicy); + + // Record every ECS task lifecycle transition (PROVISIONING → ... → STOPPED) + // so we can reconstruct a per-state timeline for scale-out latency analysis. + const taskStateEvents = new LogGroup(this, 'TaskStateEvents', { + logGroupName: `/aws/events/${ecsServiceName}-task-state`, + retention: RetentionDays.ONE_WEEK, + removalPolicy: RemovalPolicy.DESTROY, + }); + + // Use a low-level CfnRule + native resource policy rather than the L2 + // CloudWatchLogGroup target: the L2 target injects a Lambda-backed custom + // resource (an asset), which requires `cdk deploy`. This keeps the stack + // deployable via plain CloudFormation changesets. + const taskStateRule = new CfnRule(this, 'TaskStateChangeRule', { + eventPattern: { + source: ['aws.ecs'], + 'detail-type': ['ECS Task State Change'], + detail: { + clusterArn: [app.ecsService.cluster.clusterArn], + }, + }, + targets: [ + { + id: 'TaskStateEventsLog', + arn: this.formatArn({ + service: 'logs', + resource: 'log-group', + resourceName: taskStateEvents.logGroupName, + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + }), + }, + ], + }); + + new CfnResourcePolicy(this, 'TaskStateEventsPolicy', { + policyName: `${ecsServiceName}-task-state-events`, + policyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'AllowEventBridgeToLog', + Effect: 'Allow', + Principal: { Service: 'events.amazonaws.com' }, + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Resource: taskStateEvents.logGroupArn, + Condition: { + ArnEquals: { + 'aws:SourceArn': taskStateRule.attrArn, + }, + }, + }, + ], + }), + }); } } From 97c39bc5ffaa8c789a65baafa39b2052a5a48d1c Mon Sep 17 00:00:00 2001 From: Connor OMalley Date: Wed, 5 Aug 2026 16:09:04 +0100 Subject: [PATCH 06/10] Increase ECS instance cpu and memory --- .../cdk/lib/__snapshots__/renderingStack.test.ts.snap | 4 ++-- dotcom-rendering/cdk/lib/renderingStack.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap index 6e285a43350..828c33e7f72 100644 --- a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap +++ b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap @@ -2389,7 +2389,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE "VersionConsistency": "disabled", }, ], - "Cpu": "512", + "Cpu": "2048", "ExecutionRoleArn": { "Fn::GetAtt": [ "EcsTaskDefinitionExecutionRoleBE450C73", @@ -2397,7 +2397,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE ], }, "Family": "TagPageRenderingCODEEcsTaskDefinition616FF027", - "Memory": "1024", + "Memory": "4096", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", diff --git a/dotcom-rendering/cdk/lib/renderingStack.ts b/dotcom-rendering/cdk/lib/renderingStack.ts index 126a5d06fda..c54543a90fb 100644 --- a/dotcom-rendering/cdk/lib/renderingStack.ts +++ b/dotcom-rendering/cdk/lib/renderingStack.ts @@ -295,8 +295,8 @@ export class RenderingCDKStack extends CDKStack { ecsProps: { repositoryName: 'guardian/dotcom-rendering', imageIdentifier, - memoryLimitMiB: 1024, - cpu: 512, + memoryLimitMiB: 4096, + cpu: 2048, scaling: { minimumTasks: 1, maximumTasks: 9, From 5fa480b275f907a32ff4f2c07bff217f6cec31e6 Mon Sep 17 00:00:00 2001 From: Jorge Azevedo Date: Wed, 5 Aug 2026 17:08:55 +0100 Subject: [PATCH 07/10] Set target tracking to high res metric --- .../__snapshots__/renderingStack.test.ts.snap | 37 +++++++++---------- dotcom-rendering/cdk/lib/renderingStack.ts | 18 +++++++-- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap index 828c33e7f72..8fecbc9a4cf 100644 --- a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap +++ b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap @@ -1930,6 +1930,24 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE "Type": "AWS::CertificateManager::Certificate", "UpdateReplacePolicy": "Retain", }, + "CpuScalingF1B774D2": { + "Properties": { + "PolicyName": "TagPageRenderingCODECpuScalingFE034505", + "PolicyType": "TargetTrackingScaling", + "ScalingTargetId": { + "Ref": "EcsServiceTaskCountTarget02FCCE22", + }, + "TargetTrackingScalingPolicyConfiguration": { + "PredefinedMetricSpecification": { + "PredefinedMetricType": "ECSServiceAverageCPUUtilizationHighResolution", + }, + "ScaleInCooldown": 60, + "ScaleOutCooldown": 60, + "TargetValue": 50, + }, + }, + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy", + }, "DescribeEC2PolicyFF5F9295": { "Properties": { "PolicyDocument": { @@ -2236,25 +2254,6 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE }, "Type": "AWS::ApplicationAutoScaling::ScalableTarget", }, - "EcsServiceTaskCountTargetCpuScalingA9C91D4C": { - "DependsOn": [ - "EcsTaskDefinitionTaskRoleB7B6D8DD", - ], - "Properties": { - "PolicyName": "TagPageRenderingCODEEcsServiceTaskCountTargetCpuScaling4DFA1A0F", - "PolicyType": "TargetTrackingScaling", - "ScalingTargetId": { - "Ref": "EcsServiceTaskCountTarget02FCCE22", - }, - "TargetTrackingScalingPolicyConfiguration": { - "PredefinedMetricSpecification": { - "PredefinedMetricType": "ECSServiceAverageCPUUtilization", - }, - "TargetValue": 50, - }, - }, - "Type": "AWS::ApplicationAutoScaling::ScalingPolicy", - }, "EcsTaskDefinition63157ED3": { "Properties": { "ContainerDefinitions": [ diff --git a/dotcom-rendering/cdk/lib/renderingStack.ts b/dotcom-rendering/cdk/lib/renderingStack.ts index c54543a90fb..b6c3d0db234 100644 --- a/dotcom-rendering/cdk/lib/renderingStack.ts +++ b/dotcom-rendering/cdk/lib/renderingStack.ts @@ -17,7 +17,11 @@ import { Duration, RemovalPolicy, } from 'aws-cdk-lib'; -import type { ScalingInterval } from 'aws-cdk-lib/aws-applicationautoscaling'; +import type { + PredefinedMetric, + ScalingInterval, +} from 'aws-cdk-lib/aws-applicationautoscaling'; +import { TargetTrackingScalingPolicy } from 'aws-cdk-lib/aws-applicationautoscaling'; import { AdjustmentType, StepScalingPolicy } from 'aws-cdk-lib/aws-autoscaling'; import { Metric, Unit } from 'aws-cdk-lib/aws-cloudwatch'; import { SnsAction } from 'aws-cdk-lib/aws-cloudwatch-actions'; @@ -353,8 +357,16 @@ export class RenderingCDKStack extends CDKStack { 'TaskCount', ) as ScalableTaskCount; if (ecsScalableTarget) { - ecsScalableTarget.scaleOnCpuUtilization('CpuScaling', { - targetUtilizationPercent: 50, + // The high resolution predefined metric evaluates every 10s rather than 60s, + // so scaling reacts far faster. It isn't in the CDK `PredefinedMetric` enum yet. + // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/target-tracking-faster-auto-scaling.html + new TargetTrackingScalingPolicy(this, 'CpuScaling', { + scalingTarget: ecsScalableTarget, + targetValue: 50, + predefinedMetric: + 'ECSServiceAverageCPUUtilizationHighResolution' as unknown as PredefinedMetric, + scaleOutCooldown: Duration.seconds(60), + scaleInCooldown: Duration.seconds(60), }); } else { throw new Error( From 1c4baca0bf50c3aec3f7cc5e9f23c493f0403a82 Mon Sep 17 00:00:00 2001 From: Jorge Azevedo Date: Thu, 6 Aug 2026 10:04:59 +0100 Subject: [PATCH 08/10] Set CPU target to 20% --- .../cdk/lib/__snapshots__/renderingStack.test.ts.snap | 2 +- dotcom-rendering/cdk/lib/renderingStack.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap index 8fecbc9a4cf..600308d954c 100644 --- a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap +++ b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap @@ -1943,7 +1943,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE }, "ScaleInCooldown": 60, "ScaleOutCooldown": 60, - "TargetValue": 50, + "TargetValue": 20, }, }, "Type": "AWS::ApplicationAutoScaling::ScalingPolicy", diff --git a/dotcom-rendering/cdk/lib/renderingStack.ts b/dotcom-rendering/cdk/lib/renderingStack.ts index b6c3d0db234..9511a9e43a4 100644 --- a/dotcom-rendering/cdk/lib/renderingStack.ts +++ b/dotcom-rendering/cdk/lib/renderingStack.ts @@ -362,7 +362,7 @@ export class RenderingCDKStack extends CDKStack { // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/target-tracking-faster-auto-scaling.html new TargetTrackingScalingPolicy(this, 'CpuScaling', { scalingTarget: ecsScalableTarget, - targetValue: 50, + targetValue: 20, predefinedMetric: 'ECSServiceAverageCPUUtilizationHighResolution' as unknown as PredefinedMetric, scaleOutCooldown: Duration.seconds(60), From 4ae14af88320e43d32abdf2047316984a12178e6 Mon Sep 17 00:00:00 2001 From: Connor OMalley Date: Mon, 10 Aug 2026 10:46:43 +0100 Subject: [PATCH 09/10] Bump minimum size of ecs and ec2 in code --- dotcom-rendering/cdk/bin/cdk.ts | 2 +- .../cdk/lib/__snapshots__/renderingStack.test.ts.snap | 6 +++--- dotcom-rendering/cdk/lib/renderingStack.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dotcom-rendering/cdk/bin/cdk.ts b/dotcom-rendering/cdk/bin/cdk.ts index 6b5b96a3188..a6ebe0972f1 100644 --- a/dotcom-rendering/cdk/bin/cdk.ts +++ b/dotcom-rendering/cdk/bin/cdk.ts @@ -101,7 +101,7 @@ export const TagPageRenderingPropsCODE: RenderingCDKStackProps = { guApp: 'tag-page-rendering', stage: 'CODE', domainName: 'tag-page-rendering.code.dev-guardianapis.com', - scaling: { minimumInstances: 1, maximumInstances: 3 }, + scaling: { minimumInstances: 9, maximumInstances: 9 }, instanceType: InstanceType.of(InstanceClass.C8G, InstanceSize.MEDIUM), imageIdentifier: process.env.IMAGE_DIGEST ?? 'DEV', }; diff --git a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap index 600308d954c..439765b35f6 100644 --- a/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap +++ b/dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap @@ -1838,13 +1838,13 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE ], }, }, - "MaxSize": "3", + "MaxSize": "9", "MetricsCollection": [ { "Granularity": "1Minute", }, ], - "MinSize": "1", + "MinSize": "9", "Tags": [ { "Key": "App", @@ -2214,7 +2214,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE ], "Properties": { "MaxCapacity": 9, - "MinCapacity": 1, + "MinCapacity": 9, "ResourceId": { "Fn::Join": [ "", diff --git a/dotcom-rendering/cdk/lib/renderingStack.ts b/dotcom-rendering/cdk/lib/renderingStack.ts index 9511a9e43a4..3a74c7f3f7d 100644 --- a/dotcom-rendering/cdk/lib/renderingStack.ts +++ b/dotcom-rendering/cdk/lib/renderingStack.ts @@ -302,7 +302,7 @@ export class RenderingCDKStack extends CDKStack { memoryLimitMiB: 4096, cpu: 2048, scaling: { - minimumTasks: 1, + minimumTasks: 9, maximumTasks: 9, }, }, From 42375f586e713b414afabc7cd5378e00327e0f0c Mon Sep 17 00:00:00 2001 From: Connor OMalley Date: Mon, 10 Aug 2026 14:46:49 +0100 Subject: [PATCH 10/10] Startup speed tests --- startup_speed_tests/ec2_data.csv | 10 ++++ startup_speed_tests/ec2_list_instances.sh | 19 +++++++ startup_speed_tests/ec2_startup_time.sh | 69 +++++++++++++++++++++++ startup_speed_tests/ecs_data.csv | 10 ++++ startup_speed_tests/ecs_list_tasks.sh | 18 ++++++ startup_speed_tests/ecs_startup_time.sh | 64 +++++++++++++++++++++ startup_speed_tests/readme.md | 4 ++ 7 files changed, 194 insertions(+) create mode 100644 startup_speed_tests/ec2_data.csv create mode 100755 startup_speed_tests/ec2_list_instances.sh create mode 100755 startup_speed_tests/ec2_startup_time.sh create mode 100644 startup_speed_tests/ecs_data.csv create mode 100755 startup_speed_tests/ecs_list_tasks.sh create mode 100755 startup_speed_tests/ecs_startup_time.sh create mode 100644 startup_speed_tests/readme.md diff --git a/startup_speed_tests/ec2_data.csv b/startup_speed_tests/ec2_data.csv new file mode 100644 index 00000000000..7b44f9893fc --- /dev/null +++ b/startup_speed_tests/ec2_data.csv @@ -0,0 +1,10 @@ +name|startTime|readyTime +i-01491731f65174dee|2026-08-10T10:40:20+00:00|2026-08-10T10:40:45.334976Z +i-015e60fc8445464a4|2026-08-10T10:40:20+00:00|2026-08-10T10:40:45.282722Z +i-02b9afa15f24d371f|2026-08-10T10:40:20+00:00|2026-08-10T10:40:48.295906Z +i-0398aa48e6bbadc00|2026-08-10T10:40:20+00:00|2026-08-10T10:40:45.281690Z +i-049ffd956c48f5db2|2026-08-10T10:40:20+00:00|2026-08-10T10:40:38.292070Z +i-0ac7997101e84d3fb|2026-08-10T10:40:20+00:00|2026-08-10T10:40:45.281693Z +i-0d81bd6fdbac2badf|2026-08-10T10:40:20+00:00|2026-08-10T10:40:45.335295Z +i-0dd993a90055834cc|2026-08-10T10:55:07+00:00|2026-08-10T10:55:32.767845Z +i-0e18d6eac114f6620|2026-08-10T10:53:06+00:00|2026-08-10T10:53:31.478432Z diff --git a/startup_speed_tests/ec2_list_instances.sh b/startup_speed_tests/ec2_list_instances.sh new file mode 100755 index 00000000000..63283900d61 --- /dev/null +++ b/startup_speed_tests/ec2_list_instances.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +aws_command() { + aws --profile=frontend --region=eu-west-1 "$@" +} + +if [ -z "$1" ]; then + echo "Error: Please provide an ECS task ID." + echo "Usage: $0 " + exit 1 +fi + +SCALING_GROUP=$1 + +INSTANCES=$(aws_command autoscaling describe-auto-scaling-instances \ + --query "AutoScalingInstances[?AutoScalingGroupName=='$SCALING_GROUP'].InstanceId" \ + --output text) +echo $INSTANCES + diff --git a/startup_speed_tests/ec2_startup_time.sh b/startup_speed_tests/ec2_startup_time.sh new file mode 100755 index 00000000000..45ca05db595 --- /dev/null +++ b/startup_speed_tests/ec2_startup_time.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Function for AWS calls with the desired profile and region +aws_command() { + aws --profile=frontend --region=eu-west-1 "$@" +} + +if [ -z "$1" ]; then + echo "Error: Please provide an EC2 instance ID." + echo "Usage: $0 " + exit 1 +fi + +INSTANCE_ID=$1 + +# Get the EC2 instance launch time +LAUNCH_TIME=$(aws_command ec2 describe-instances \ + --instance-ids "$INSTANCE_ID" \ + --query 'Reservations[*].Instances[*].LaunchTime' \ + --output text) + +PRIVATE_IP=$(aws_command ec2 describe-instances \ + --instance-ids "$INSTANCE_ID" \ + --query 'Reservations[*].Instances[*].PrivateIpAddress' \ + --output text) + +if [ -z "$LAUNCH_TIME" ]; then + echo "Error: Unable to find launch time for instance $INSTANCE_ID." + exit 1 +fi + +# Calculate 10 minutes after the launch time (ISO 8601 DateTime format) +LAUNCH_TIME_EPOCH_MILLISECONDS=$(python3 -c " +from datetime import datetime +import sys +dt = sys.argv[1].replace('Z', '+00:00') +print(int(datetime.fromisoformat(dt).timestamp() * 1000)) +" "$LAUNCH_TIME") + +END_LOGS_TIME_EPOCH_MILLISECONDS=$(python3 -c " +from datetime import datetime, timedelta +import sys +dt = sys.argv[1].replace('Z', '+00:00') +launch_time = datetime.fromisoformat(dt) +end_time = launch_time + timedelta(minutes=10) +print(int(end_time.timestamp() * 1000)) +" "$LAUNCH_TIME") + +# Get logs from the specific log group and log stream prefix +LOG_GROUP_NAME="/aws/elasticloadbalancing/tag-page-rendering-CODE-health-check" +LOG_STREAM_PREFIX="ALB_Health_Check_Logs" + +# Find the first health check status that is "healthy" +JSON_RESULTS=$(aws_command logs filter-log-events \ + --log-group-name "$LOG_GROUP_NAME" \ + --log-stream-name-prefix "$LOG_STREAM_PREFIX" \ + --start-time $LAUNCH_TIME_EPOCH_MILLISECONDS \ + --end-time $END_LOGS_TIME_EPOCH_MILLISECONDS) + +FIRST_POSITIVE_HEALTH_CHECK_TIME=$(echo $JSON_RESULTS | jq -r --arg ip "${PRIVATE_IP}:9000" ' + .events + | map(.message | fromjson | select(.target_addr == $ip)) + | sort_by(.time) + | map(select(.status == "PASS")) + | .[0]["time"] +') +echo $INSTANCE_ID +echo $LAUNCH_TIME +echo $FIRST_POSITIVE_HEALTH_CHECK_TIME diff --git a/startup_speed_tests/ecs_data.csv b/startup_speed_tests/ecs_data.csv new file mode 100644 index 00000000000..e1f3fa40f1b --- /dev/null +++ b/startup_speed_tests/ecs_data.csv @@ -0,0 +1,10 @@ +name|startTime|readyTime +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/335f73cc3d5a494d9b19160647009002|2026-08-10T11:42:53.902000+01:00|2026-08-10T10:43:21.806077Z +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/3716bacda7d24158be3a5ec2c27eec99|2026-08-10T11:42:53.902000+01:00|2026-08-10T10:43:21.810286Z +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/8f100ba08eb54c09a4fef11c7041cba4|2026-08-10T11:42:53.902000+01:00|2026-08-10T10:43:21.808961Z +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/ba5a248a984647e28f97be02e7449b19|2026-08-10T11:42:53.902000+01:00|2026-08-10T10:43:21.809508Z +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/cdad2a228ea8478da3f3b9eccabeafdb|2026-08-10T11:42:53.902000+01:00|2026-08-10T10:43:21.806751Z +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/d90f6341594b4613897c6e20ec3de31d|2026-08-10T11:40:27.294000+01:00|2026-08-10T10:40:57.975687Z +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/f0235eaabe0a472eb6c1afae5dedcbe0|2026-08-10T11:42:53.902000+01:00|2026-08-10T10:43:31.565310Z +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/f5ff52fb9d98445d95dc410b49757385|2026-08-10T11:42:53.902000+01:00|2026-08-10T10:43:31.565995Z +arn:aws:ecs:eu-west-1:642631414762:task/frontend-tag-page-rendering-CODE-tagpagerenderingEcsClusterE7696595-TylfDV8MtwHi/fd8cfa9cd5014df89f72af4b0022105f|2026-08-10T11:42:53.902000+01:00|2026-08-10T10:43:21.809905Z diff --git a/startup_speed_tests/ecs_list_tasks.sh b/startup_speed_tests/ecs_list_tasks.sh new file mode 100755 index 00000000000..3a4edc37f9e --- /dev/null +++ b/startup_speed_tests/ecs_list_tasks.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +aws_command() { + aws --profile=frontend --region=eu-west-1 "$@" +} + +if [ -z "$1" ]; then + echo "Error: Please provide an ECS cluster." + echo "Usage: $0 " + exit 1 +fi + +CLUSTER=$1 + +aws_command ecs list-tasks \ + --cluster $CLUSTER \ + --query 'taskArns[*]' \ + --output text diff --git a/startup_speed_tests/ecs_startup_time.sh b/startup_speed_tests/ecs_startup_time.sh new file mode 100755 index 00000000000..fb056128dcf --- /dev/null +++ b/startup_speed_tests/ecs_startup_time.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +aws_command() { + aws --profile=frontend --region=eu-west-1 "$@" +} + +if [ -z "$1" ]; then + echo "Error: Please provide an ECS task ID." + echo "Usage: $0 " + exit 1 +fi + +TASK_ID=$1 +CLUSTER=$2 + +# Get the ECS details +CONTAINER_DETAILS=$(aws_command ecs describe-tasks \ + --tasks "$TASK_ID" \ + --cluster "$CLUSTER" \ + --output json \ + --query 'tasks[] | [].[{startedAt: startedAt, createdAt: createdAt, taskArn: taskArn, ipAddr: containers[0].networkInterfaces[0].privateIpv4Address}]') + +LAUNCH_TIME=$(echo $CONTAINER_DETAILS | jq -r '.[0].[0].createdAt') +PRIVATE_IP=$(echo $CONTAINER_DETAILS | jq -r '.[0].[0].ipAddr') + +# Calculate 10 minutes after the launch time (ISO 8601 DateTime format) +LAUNCH_TIME_EPOCH_MILLISECONDS=$(python3 -c " +from datetime import datetime +import sys +dt = sys.argv[1].replace('Z', '+00:00') +print(int(datetime.fromisoformat(dt).timestamp() * 1000)) +" "$LAUNCH_TIME") + +END_LOGS_TIME_EPOCH_MILLISECONDS=$(python3 -c " +from datetime import datetime, timedelta +import sys +dt = sys.argv[1].replace('Z', '+00:00') +launch_time = datetime.fromisoformat(dt) +end_time = launch_time + timedelta(minutes=10) +print(int(end_time.timestamp() * 1000)) +" "$LAUNCH_TIME") + +# Get logs from the specific log group and log stream prefix +LOG_GROUP_NAME="/aws/elasticloadbalancing/tag-page-rendering-CODE-health-check" +LOG_STREAM_PREFIX="ALB_Health_Check_Logs" + +# Find the first health check status that is "healthy" +JSON_RESULTS=$(aws_command logs filter-log-events \ + --log-group-name "$LOG_GROUP_NAME" \ + --log-stream-name-prefix "$LOG_STREAM_PREFIX" \ + --start-time $LAUNCH_TIME_EPOCH_MILLISECONDS \ + --end-time $END_LOGS_TIME_EPOCH_MILLISECONDS) + +FIRST_POSITIVE_HEALTH_CHECK_TIME=$(echo $JSON_RESULTS | jq -r --arg ip "${PRIVATE_IP}:9000" ' + .events + | map(.message | fromjson | select(.target_addr == $ip)) + | sort_by(.time) + | map(select(.status == "PASS")) + | .[0]["time"] +') + +echo $TASK_ID +echo $LAUNCH_TIME +echo $FIRST_POSITIVE_HEALTH_CHECK_TIME diff --git a/startup_speed_tests/readme.md b/startup_speed_tests/readme.md new file mode 100644 index 00000000000..533639f3c7b --- /dev/null +++ b/startup_speed_tests/readme.md @@ -0,0 +1,4 @@ +``` +duckdb -c "select *, readyTime - startTime from ec2_data.csv" +duckdb -c "select *, readyTime - startTime from ecs_data.csv" +```