diff --git a/backend/.env.example b/backend/.env.example index 204695090..f17486001 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -13,6 +13,7 @@ APP_ALLOW_FIRST_TIME_REGISTER= # -- BACKEND -- BACKEND_URL=http://localhost:5000 +CELERY_BROKER=redis # -- FRONTEND -- FRONTEND_URL=http://localhost:8090 diff --git a/backend/.env.test b/backend/.env.test index dbd9223ba..786fb0e31 100644 --- a/backend/.env.test +++ b/backend/.env.test @@ -7,6 +7,7 @@ LOG_USE_STREAM=1 # -- BACKEND -- BACKEND_URL=http://localhost:5000 +CELERY_BROKER=redis # -- FRONTEND -- FRONTEND_URL=http://localhost:8090 diff --git a/backend/requirements.txt b/backend/requirements.txt index 2351402f0..2e3b81160 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,7 +3,7 @@ argon2-cffi==25.* argon2-cffi-bindings==25.* Babel==2.* caldav==1.6.0 -celery[redis]==5.* +celery[sqs]==5.* cryptography==46.0.6 dnspython==2.* fastapi[standard]==0.* diff --git a/backend/src/appointment/celery_app.py b/backend/src/appointment/celery_app.py index 6444da263..ed25bacb3 100644 --- a/backend/src/appointment/celery_app.py +++ b/backend/src/appointment/celery_app.py @@ -20,17 +20,23 @@ def _base_redis_url() -> str: def create_celery_app() -> Celery: load_dotenv() - redis_url = _base_redis_url() - - broker_url = '/'.join(filter(None, [os.getenv('CELERY_BROKER') or redis_url, os.getenv('REDIS_CELERY_DB')])) - result_backend = '/'.join( - filter(None, [os.getenv('CELERY_BACKEND') or redis_url, os.getenv('REDIS_CELERY_RESULTS_DB')]) - ) - - if broker_url.startswith('rediss://'): - broker_url = f'{broker_url}?ssl_cert_reqs=CERT_REQUIRED' - if result_backend.startswith('rediss://'): - result_backend = f'{result_backend}?ssl_cert_reqs=CERT_REQUIRED' + _celery_broker = os.getenv('CELERY_BROKER', 'redis') + if _celery_broker == 'redis': + redis_url = _base_redis_url() + + broker_url = '/'.join(filter(None, [os.getenv('CELERY_BROKER') or redis_url, os.getenv('REDIS_CELERY_DB')])) + result_backend = '/'.join( + filter(None, [os.getenv('CELERY_BACKEND') or redis_url, os.getenv('REDIS_CELERY_RESULTS_DB')]) + ) + + if broker_url.startswith('rediss://'): + broker_url = f'{broker_url}?ssl_cert_reqs=CERT_REQUIRED' + if result_backend.startswith('rediss://'): + result_backend = f'{result_backend}?ssl_cert_reqs=CERT_REQUIRED' + elif _celery_broker == 'sqs': + + else: + raise ValueError(f'"{_celery_broker}" is not a valid value for CELERY_BROKER. Choose "redis" or "sqs".') result_expires = 3600 task_always_eager = os.getenv('CELERY_EAGER', 'False') == 'True' diff --git a/docker-compose.yml b/docker-compose.yml index d924bd163..8c9faecdb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - ./backend/.env:/app/.env - ./backend/src/appointment:/app/appointment environment: + - CELERY_BROKER=redis - CONTAINER_ROLE=api - IS_LOCAL_DEV=yes - RELEASE_VERSION=localdev @@ -23,6 +24,7 @@ services: <<: *backend ports: [] environment: + - CELERY_BROKER=redis - CONTAINER_ROLE=worker depends_on: - backend @@ -32,6 +34,7 @@ services: <<: *backend ports: [] environment: + - CELERY_BROKER=redis - CONTAINER_ROLE=beat depends_on: - backend @@ -42,6 +45,7 @@ services: ports: - 5556:5555 environment: + - CELERY_BROKER=redis - CONTAINER_ROLE=flower - FLOWER_UNAUTHENTICATED_API=true depends_on: diff --git a/pulumi/__main__.py b/pulumi/__main__.py index 28fcdf863..6e0aa5e79 100644 --- a/pulumi/__main__.py +++ b/pulumi/__main__.py @@ -13,6 +13,7 @@ from fargate import fargate from redis import redis_cache from security_groups import security_groups +from sqs import sqs #: Environments in which we should build a proxy for Posthog calls POSTHOG_PROXY_STACKS = ['prod'] @@ -91,6 +92,13 @@ for afc_name, afc_config in resources.get('tb:fargate:AutoscalingFargateCluster', {}).items() } +# Celery does not currently support clustered Redis brokers. Therefore, we build an SQS queue to route tasks through. +sqs( + project=project, + api_exec_role=fargate_clusters['backend'].resources['task_role'].name, + celery_exec_role=afcs['appointment'].resources['execution_roles']['celery'].name, +) + # CloudFront function to handle request rewrites headed to the backend rewrite_function = cloudfront.rewrite_function(project=project) project.resources['cf_rewrite_function'] = rewrite_function diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index fb93f6b31..b90bc1983 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -3,21 +3,25 @@ ### Special variables used throughout this file # Update this value to update all containers based on the thunderbird/appointment image -.apmt_image: &APMT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/appointment:85f15a97bf5ee889bb3ca56cea3f4dac7f5c00d2 +.apmt_image: &APMT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/appointment:f0b6b209385f77dd2d522ded8e4826b66eb045df # These variables are common to Accounts application environments. Some tasks will require additional configuration. .app_env: &VAR_APP_ENV {name: "APP_ENV", value: "dev"} .auth_scheme: &VAR_AUTH_SCHEME {name: "AUTH_SCHEME", value: "oidc"} +.celery_broken: &VAR_CELERY_BROKER {name: "CELERY_BROKER", value: "sqs"} .database_engine: &VAR_DATABASE_ENGINE {name: "DATABASE_ENGINE", value: "postgresql"} .database_port: &VAR_DATABASE_PORT {name: "DATABASE_PORT", value: "5432"} +.frontend_url: &VAR_FRONTEND_URL {name: "FRONTEND_URL", value: "https://appointment-dev.tb.pro"} +.fxa_callback: &VAR_FXA_CALLBACK {name: "FXA_CALLBACK", value: "https://appointment-dev.tb.pro/fxa"} +.google_auth_callback: &VAR_GOOGLE_AUTH_CALLBACK {name: "GOOGLE_AUTH_CALLBACK", value: "https://appointment-dev.tb.pro/api/v1/google/callback"} .jwt_algo: &VAR_JWT_ALGO {name: "JWT_ALGO", value: "HS256"} .jwt_expire_in_mins: &VAR_JWT_EXPIRE_IN_MINS {name: "JWT_EXPIRE_IN_MINS", value: "10000"} .log_level: &VAR_LOG_LEVEL {name: "LOG_LEVEL", value: "ERROR"} .log_use_stream: &VAR_LOG_USE_STREAM {name: "LOG_USE_STREAM", value: "True"} -.oids_exp_grace_period: &VAR_OIDC_EXP_GRACE_PERIOD {name: "OIDC_EXP_GRACE_PERIOD", value: "60"} +.oidc_exp_grace_period: &VAR_OIDC_EXP_GRACE_PERIOD {name: "OIDC_EXP_GRACE_PERIOD", value: "60"} .oidc_fallback_match_by_email: &VAR_OIDC_FALLBACK_MATCH_BY_EMAIL {name: "OIDC_FALLBACK_MATCH_BY_EMAIL", value: "True"} .oidc_token_introspection_url: &VAR_OIDC_TOKEN_INTROSPECTION_URL {name: "OIDC_TOKEN_INTROSPECTION_URL", value: "https://auth-dev.tb.pro/realms/tbpro/protocol/openid-connect/token/introspect"} -.frontend_url: &VAR_FRONTEND_URL {name: "FRONTEND_URL", value: "https://appointment-dev.tb.pro"} +.posthog_host: &VAR_POSTHOG_HOST {name: "POSTHOG_HOST", value: "https://us.i.posthog.com"} .redis_db: &VAR_REDIS_DB {name: "REDIS_DB", value: "0"} .redis_port: &VAR_REDIS_PORT {name: "REDIS_PORT", value: "6379"} .redis_use_ssl: &VAR_REDIS_USE_SSL {name: "REDIS_USE_SSL", value: "True"} @@ -26,20 +30,43 @@ .sentry_dsn: &VAR_SENTRY_DSN {name: "SENTRY_DSN", value: "https://5dddca3ecc964284bb8008bc2beef808@o4505428107853824.ingest.sentry.io/4505428124827648"} .service_email: &VAR_SERVICE_EMAIL {name: "SERVICE_EMAIL", value: "no-reply@appointment-dev.tb.pro"} .short_base_url: &VAR_SHORT_BASE_URL {name: "SHORT_BASE_URL", value: "https://dev.apt.mt"} +.smtp_port: &VAR_SMTP_PORT {name: "SMTP_PORT", value: "587"} +.smtp_security: &VAR_SMTP_SECURITY {name: "SMTP_SECURITY", value: "STARTTLS"} .tb_accounts_caldav_url: &VAR_TB_ACCOUNTS_CALDAV_URL {name: "TB_ACCOUNTS_CALDAV_URL", value: "https://dev-thundermail.com"} +.tb_accounts_host: &VAR_TB_ACCOUNTS_HOST {name: "TB_ACCOUNTS_HOST", value: "https://accounts-dev.tb.pro"} .tier_basic_calendar_limit: &VAR_TIER_BASIC_CALENDAR_LIMIT {name: "TIER_BASIC_CALENDAR_LIMIT", value: "3"} .tier_plus_calendar_limit: &VAR_TIER_PLUS_CALENDAR_LIMIT {name: "TIER_PLUS_CALENDAR_LIMIT", value: "5"} .tier_pro_calendar_limit: &VAR_TIER_PRO_CALENDAR_LIMIT {name: "TIER_PRO_CALENDAR_LIMIT", value: "10"} .zoom_api_enabled: &VAR_ZOOM_API_ENABLED {name: "ZOOM_API_ENABLED", value: "True"} +.zoom_api_new_app: &VAR_ZOOM_API_NEW_APP {name: "ZOOM_API_NEW_APP", value: "False"} .zoom_auth_callback: &VAR_ZOOM_AUTH_CALLBACK {name: "ZOOM_AUTH_CALLBACK", value: "https://appointment-dev.tb.pro/api/v1/zoom/callback"} # These variables are also common to our environments, but are pulled from secret stores instead +.app_admin_allow_list: &SECRET_APP_ADMIN_ALLOW_LIST {name: "APP_ADMIN_ALLOW_LIST", valueFrom: ""} +.appointment_caldav_secret: &SECRET_APPOINTMENT_CALDAV_SECRET {name: "APPOINTMENT_CALDAV_SECRET", valueFrom: ""} .database_host: &SECRET_DATABASE_HOST {name: "DATABASE_HOST", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-host-pnM2d9"} .database_name: &SECRET_DATABASE_NAME {name: "DATABASE_NAME", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-name-slQJtj"} -.database_username: &SECRET_DATABASE_USERNAME {name: "DATABASE_USERNAME", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-user-9EgJWM"} .database_password: &SECRET_DATABASE_PASSWORD {name: "DATABASE_PASSWORD", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-password-yaGZdw"} +.database_username: &SECRET_DATABASE_USERNAME {name: "DATABASE_USERNAME", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-user-9EgJWM"} +.db_secret: &SECRET_DB_SECRET {name: "DB_SECRET", valueFrom: ""} +.fxa_allow_list: &SECRET_FXA_ALLOW_LIST {name: "FXA_ALLOW_LIST", valueFrom: ""} +.fxa_client_id: &SECRET_FXA_CLIENT_ID {name: "FXA_CLIENT_ID", valueFrom: ""} +.fxa_open_id_config: &SECRET_FXA_OPEN_ID_CONFIG {name: "FXA_OPEN_ID_CONFIG", valueFrom: ""} +.fxa_secret: &SECRET_FXA_SECRET {name: "FXA_SECRET", valueFrom: ""} +.google_auth_client_id: &SECRET_GOOGLE_AUTH_CLIENT_ID {name: "GOOGLE_AUTH_CLIENT_ID", valueFrom: ""} +.google_auth_project_id: &SECRET_GOOGLE_AUTH_PROJECT_ID {name: "GOOGLE_AUTH_PROJECT_ID", valueFrom: ""} +.google_auth_secret: &SECRET_GOOGLE_AUTH_SECRET {name: "GOOGLE_AUTH_SECRET", valueFrom: ""} +.jwt_secret: &SECRET_JWT_SECRET {name: "JWT_SECRET", valueFrom: ""} .oidc_client_id: &SECRET_OIDC_CLIENT_ID {name: "OIDC_CLIENT_ID", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/oidc-client-id-sbebkz"} .oidc_client_secret: &SECRET_OIDC_CLIENT_SECRET {name: "OIDC_CLIENT_SECRET", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/oidc-client-secret-I9BTf2"} +.posthog_project_key: &SECRET_POSTHOG_PROJECT_KEY {name: "POSTHOG_PROJECT_KEY", valueFrom: ""} +.session_secret: &SECRET_SESSION_SECRET {name: "SESSION_SECRET", valueFrom: ""} +.signed_secret: &SECRET_SIGNED_SECRET {name: "SIGNED_SECRET", valueFrom: ""} +.smtp_pass: &SECRET_SMTP_PASS {name: "SMTP_PASS", valueFrom: ""} +.smtp_url: &SECRET_SMTP_URL {name: "SMTP_URL", valueFrom: ""} +.smtp_user: &SECRET_SMTP_USER {name: "SMTP_USER", valueFrom: ""} +.zoom_api_secret: &SECRET_ZOOM_API_SECRET {name: "ZOOM_API_SECRET", valueFrom: ""} +.zoom_auth_client_id: &SECRET_ZOOM_AUTH_CLIENT_ID {name: "ZOOM_AUTH_CLIENT_ID", valueFrom: ""} # This forms the base of Appointment-based task definitions. The blank fields commented here must be set on any # inheriting task definition: @@ -502,8 +529,8 @@ resources: autoscalers: celery: - min_capacity: 1 - max_capacity: 1 + min_capacity: 0 + max_capacity: 0 flower: min_capacity: 1 max_capacity: 1 diff --git a/pulumi/config.prod.yaml b/pulumi/config.prod.yaml index 42421f38d..5143c1d13 100644 --- a/pulumi/config.prod.yaml +++ b/pulumi/config.prod.yaml @@ -8,6 +8,7 @@ # These variables are common to Accounts application environments. Some tasks will require additional configuration. .app_env: &VAR_APP_ENV {name: "APP_ENV", value: "prod"} .auth_scheme: &VAR_AUTH_SCHEME {name: "AUTH_SCHEME", value: "oidc"} +.celery_broker: &VAR_CELERY_BROKER {name: "CELERY_BROKER", value: "sqs"} .database_engine: &VAR_DATABASE_ENGINE {name: "DATABASE_ENGINE", value: "postgresql"} .database_port: &VAR_DATABASE_PORT {name: "DATABASE_PORT", value: "5432"} .frontend_url: &VAR_FRONTEND_URL {name: "FRONTEND_URL", value: "https://appointment.tb.pro"} diff --git a/pulumi/config.stage.yaml b/pulumi/config.stage.yaml index 3ce95a449..4267cdc55 100644 --- a/pulumi/config.stage.yaml +++ b/pulumi/config.stage.yaml @@ -8,6 +8,7 @@ # These variables are common to Accounts application environments. Some tasks will require additional configuration. .app_env: &VAR_APP_ENV {name: "APP_ENV", value: "stage"} .auth_scheme: &VAR_AUTH_SCHEME {name: "AUTH_SCHEME", value: "oidc"} +.celery_broken: &VAR_CELERY_BROKER {name: "CELERY_BROKER", value: "sqs"} .database_engine: &VAR_DATABASE_ENGINE {name: "DATABASE_ENGINE", value: "postgresql"} .database_port: &VAR_DATABASE_PORT {name: "DATABASE_PORT", value: "5432"} .frontend_url: &VAR_FRONTEND_URL {name: "FRONTEND_URL", value: "https://appointment-stage.tb.pro"} diff --git a/pulumi/sqs.py b/pulumi/sqs.py new file mode 100644 index 000000000..ae671ae55 --- /dev/null +++ b/pulumi/sqs.py @@ -0,0 +1,184 @@ +import json +import pulumi +import pulumi_aws as aws +import tb_pulumi.secrets + + +def sqs(project, api_exec_role, celery_exec_role): + # Build a queue + celery_sqs_queue = aws.sqs.Queue( + f'{project.name_prefix}-queue-celery', + fifo_queue=True, + name=f'{project.name_prefix}-celery.fifo', # FIFO queue names *must* end in ".fifo" + # policy=json.dumps( + # { + # 'Version': '2012-10-17', + # 'Id': 'OwnerAccessPolicy', + # 'Statement': [ + # { + # 'Sid': 'OwnerAccessStatement', + # 'Effect': 'Allow', + # 'Principal': {'AWS': '768512802988'}, + # 'Action': ['SQS:*'], + # 'Resource': 'arn:aws:sqs:eu-central-1:768512802988:rjung-test', + # } + # ], + # } + # ), + tags=project.common_tags, + ) + + # Build IAM policies allowing receipt and publication of messages to the queue + iam_read_policy_doc = celery_sqs_queue.arn.apply( + lambda queue_arn: json.dumps( + { + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': 'AllowSQSReceipt', + 'Effect': 'Allow', + 'Action': [ + 'sqs:ChangeMessageVisibility', + 'sqs:DeleteMessage', + 'sqs:ReceiveMessage', + ], + 'Resource': [ + queue_arn, + ], + } + ], + } + ) + ) + + iam_queue_read_access_policy = aws.iam.Policy( + f'{project.name_prefix}-iampolicy-celerysqs-read', + description=f'Allow subscription to the {project.name_prefix}-celery SQS queue', + name=f'{project.name_prefix}-celery-read', + policy=iam_read_policy_doc, + tags=project.common_tags, + opts=pulumi.ResourceOptions(depends_on=[celery_sqs_queue]), + ) + + iam_write_policy_doc = celery_sqs_queue.arn.apply( + lambda queue_arn: json.dumps( + { + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': 'AllowSQSPublication', + 'Effect': 'Allow', + 'Action': [ + 'sqs:SendMessage', + ], + 'Resource': [queue_arn], + } + ], + } + ) + ) + + iam_queue_write_access_policy = aws.iam.Policy( + f'{project.name_prefix}-iampolicy-celerysqs-write', + description=f'Allow publication to the {project.name_prefix}-celery SQS queue', + name=f'{project.name_prefix}-celery-write', + policy=iam_write_policy_doc, + tags=project.common_tags, + opts=pulumi.ResourceOptions(depends_on=[celery_sqs_queue]), + ) + + # Attach those policies to the appropriate containers + aws.iam.RolePolicyAttachment( + f'{project.name_prefix}-polatt-celeryread', + policy_arn=iam_queue_read_access_policy.arn, + role=celery_exec_role, + opts=pulumi.ResourceOptions(depends_on=[iam_queue_read_access_policy]), + ) + + aws.iam.RolePolicyAttachment( + f'{project.name_prefix}-polatt-celerywrite', + policy_arn=iam_queue_write_access_policy.arn, + role=celery_exec_role, + opts=pulumi.ResourceOptions(depends_on=[iam_queue_write_access_policy]), + ) + + aws.iam.RolePolicyAttachment( + f'{project.name_prefix}-polatt-backendwrite', + policy_arn=iam_queue_write_access_policy.arn, + role=api_exec_role, + opts=pulumi.ResourceOptions(depends_on=[iam_queue_write_access_policy]), + ) + + # Celery won't let us use container/IMDS permissions; we must supply access key/ID options. + # Therefore, we must build users with access keys to assign permissions to our running Celery configs. + read_user = aws.iam.User( + f'{project.name_prefix}-user', + name=f'{project.name_prefix}-celery-sqs-read-user', + path='/', + tags=project.common_tags, + ) + + read_key_blue = aws.iam.AccessKey( + f'{project.name_prefix}-celery-sqs-read-key', + user = read_user, + status='Active', + ) + + # read_key_green = aws.iam.AccessKey( # Uncomment when we need to rotate credentials + # user=read_user, + # status='Inactive', + # ) + + write_user = aws.iam.User( + f'{project.name_prefix}-celery-sqs-write-user', + name=f'{project.name_prefix}-celery-sqs-write', + path='/', + tags=project.common_tags, + ) + + write_key_blue = aws.iam.AccessKey( + f'{project.name_prefix}-celery-sqs-write-key', + user = write_user, + status='Active', + ) + + # write_key_green = aws.iam.AccessKey( # Uncomment when we need to rotate credentials + # user=write_user, + # status='Inactive', + # ) + + aws.iam.UserPolicyAttachment( + f'{project.name_prefix}-polatt-celerysqsreader-read', + policy_arn=iam_queue_read_access_policy, + user=read_user, + opts=pulumi.ResourceOptions(depends_on=[iam_queue_read_access_policy, read_user]) + ) + + aws.iam.UserPolicyAttachment( + f'{project.name_prefix}-polatt-celerysqswriter-read', + policy_arn=iam_queue_read_access_policy, + user=write_user, + opts=pulumi.ResourceOptions(depends_on=[iam_queue_read_access_policy, read_user]) + ) + + aws.iam.UserPolicyAttachment( + f'{project.name_prefix}-polatt-celerysqswriter-write', + policy_arn=iam_queue_write_access_policy, + user=write_user, + opts=pulumi.ResourceOptions(depends_on=[iam_queue_read_access_policy, read_user]) + ) + + # Build secrets containing these keys' super secret data + tb_pulumi.secrets.SecretsManagerSecret( + f'{project.name_prefix}-secret-celerysqsread-blue', + project=project, + secret_name=f'{project.project}/{project.stack}/celerysqsreaduser-blue', + secret_value={'access_key_id': read_key_blue.id, 'secret_access_key': read_key_blue.secret} + ) + + tb_pulumi.secrets.SecretsManagerSecret( + f'{project.name_prefix}-secret-celerysqswrite-blue', + project=project, + secret_name=f'{project.project}/{project.stack}/celerysqswriteuser-blue', + secret_value={'access_key_id': read_key_blue.id, 'secret_access_key': read_key_blue.secret} + ) \ No newline at end of file