comptime STACK_TEMPLATE = StringSpan("# orca on AWS with the ec2 runner (DESIGN.md, phase 5, PR 5b): one stack,\n# so it's reviewed as one file and deleted as one stack.\n#\n# - a VPC with one public subnet in the Availability Zone given, and a\n# gateway endpoint for S3;\n# - the control instance: the scheduler (ORCA_RUNNER=ec2), Postgres and\n# orca ui as containers, Caddy in front with a Let's Encrypt certificate;\n# in one of two slots, blue and green, so that orca aws replace-control\n# can boot a new one from this template beside it and move the Elastic\n# IP (DESIGN.md, \22Replacing the control instance\22);\n# - what runs need: a bucket for the warehouse and logs, a repository for\n# the image, a role and profile for run instances, a security group they\n# share;\n# - a bucket for the lake, mlake-public-<account>: the warehouse,\n# ORCA_WAREHOUSE, is in it; orca's own bucket keeps the logs. It's private\n# unless PublicData is true, when anyone may read it unsigned (the\n# project's resource control policy refuses other accounts' signed\n# requests), billed to the project at S3's egress price;\n# - the console's hostname, served by the same Caddy from a container\n# named console on port 8080, once one runs;\n# - a Budgets alert.\n#\n# orca ui is public for reading, as a demo is; its actions need a login\n# (orca user add, in the scheduler's container). Nothing else accepts a\n# connection from outside: Postgres takes the run instances' only, and the\n# instances take each other's only.\nAWSTemplateFormatVersion: \222010-09-09\22\nDescription: orca with the ec2 runner, a control instance and runs on instances of their own\n\nParameters:\n AvailabilityZone:\n Type: AWS::EC2::AvailabilityZone::Name\n Default: us-east-2a\n ControlInstanceType:\n Type: String\n Default: t4g.large\n AllowedRunTypes:\n Type: CommaDelimitedList\n Default: \22c7g.large,c7g.xlarge,c7g.2xlarge,c7g.4xlarge,t4g.medium,t4g.large\22\n Description: The only instance types the scheduler may start\n Ami:\n Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>\n Default: /aws/service/ecs/optimized-ami/amazon-linux-2023/arm64/recommended/image_id\n Description: arm64 Amazon Linux 2023 with Docker, for the control and run instances\n ImageTag:\n Type: String\n Default: latest\n Description: The tag of the orca image the control instance starts with (orca aws push pushes latest)\n ConsoleHostname:\n Type: String\n Default: libs.mlake.dev\n Description: The console app's hostname, an A record to PublicAddress; empty for none\n Hostname:\n Type: String\n Default: orca-demo.mlake.dev\n Description: The UI's public name, an A record to PublicAddress; it answers at <PublicAddress>.sslip.io too\n DbPassword:\n Type: String\n NoEcho: true\n MinLength: 16\n # It goes into a shell script and a URL as it is.\n AllowedPattern: \22[A-Za-z0-9]+\22\n BudgetEmail:\n Type: String\n PublicData:\n Type: String\n Default: \22false\22\n AllowedValues: [\22true\22, \22false\22]\n Description: Whether anyone may read the lake's bucket unsigned; downloads are billed to the project\n BudgetDollars:\n Type: Number\n Default: 100\n ControlEnv:\n Type: String\n Default: \22\22\n # It goes into a shell script in single quotes.\n AllowedPattern: \22[^']*\22\n Description: The control instance's settings beyond the template's, KEY=VALUE entries separated by \22;\22 (orca aws up --env); never a secret\n ActiveControl:\n Type: String\n Default: blue\n AllowedValues: [blue, green]\n Description: The control slot the Elastic IP is on (orca aws replace-control moves it)\n BlueSlot:\n Type: String\n Default: \22true\22\n AllowedValues: [\22true\22, \22false\22]\n Description: Whether slot blue (Control) has an instance\n GreenSlot:\n Type: String\n Default: \22false\22\n AllowedValues: [\22true\22, \22false\22]\n Description: Whether slot green (ControlGreen) has an instance\n BlueAmi:\n Type: String\n Default: \22\22\n Description: Slot blue's image id, pinned; empty for Ami's\n GreenAmi:\n Type: String\n Default: \22\22\n Description: Slot green's image id, pinned; empty for Ami's\n\nRules:\n ActiveSlotHasAnInstance:\n Assertions:\n - Assert: !Or\n - !And [!Equals [!Ref ActiveControl, blue], !Equals [!Ref BlueSlot, \22true\22]]\n - !And [!Equals [!Ref ActiveControl, green], !Equals [!Ref GreenSlot, \22true\22]]\n AssertDescription: The slot ActiveControl names has to have an instance (BlueSlot or GreenSlot true)\n\nConditions:\n Public: !Equals [!Ref PublicData, \22true\22]\n BlueOn: !Equals [!Ref BlueSlot, \22true\22]\n GreenOn: !Equals [!Ref GreenSlot, \22true\22]\n GreenActive: !Equals [!Ref ActiveControl, green]\n BlueAmiPinned: !Not [!Equals [!Ref BlueAmi, \22\22]]\n GreenAmiPinned: !Not [!Equals [!Ref GreenAmi, \22\22]]\n\nResources:\n Vpc:\n Type: AWS::EC2::VPC\n Properties:\n CidrBlock: 10.42.0.0/16\n EnableDnsHostnames: true\n EnableDnsSupport: true\n Tags: [{Key: Name, Value: orca}]\n Subnet:\n Type: AWS::EC2::Subnet\n Properties:\n VpcId: !Ref Vpc\n CidrBlock: 10.42.0.0/20\n AvailabilityZone: !Ref AvailabilityZone\n MapPublicIpOnLaunch: true\n Tags: [{Key: Name, Value: orca}]\n Gateway:\n Type: AWS::EC2::InternetGateway\n GatewayAttachment:\n Type: AWS::EC2::VPCGatewayAttachment\n Properties: {VpcId: !Ref Vpc, InternetGatewayId: !Ref Gateway}\n Routes:\n Type: AWS::EC2::RouteTable\n Properties: {VpcId: !Ref Vpc}\n DefaultRoute:\n Type: AWS::EC2::Route\n DependsOn: GatewayAttachment\n Properties: {RouteTableId: !Ref Routes, DestinationCidrBlock: 0.0.0.0/0, GatewayId: !Ref Gateway}\n SubnetRoutes:\n Type: AWS::EC2::SubnetRouteTableAssociation\n Properties: {SubnetId: !Ref Subnet, RouteTableId: !Ref Routes}\n # S3 traffic stays in the region, and costs nothing.\n S3Endpoint:\n Type: AWS::EC2::VPCEndpoint\n Properties:\n VpcId: !Ref Vpc\n ServiceName: !Sub com.amazonaws.${AWS::Region}.s3\n VpcEndpointType: Gateway\n RouteTableIds: [!Ref Routes]\n\n Bucket:\n Type: AWS::S3::Bucket\n Properties:\n BucketName: !Sub orca-${AWS::AccountId}-${AWS::Region}\n PublicAccessBlockConfiguration:\n BlockPublicAcls: true\n BlockPublicPolicy: true\n IgnorePublicAcls: true\n RestrictPublicBuckets: true\n BucketEncryption:\n ServerSideEncryptionConfiguration:\n - ServerSideEncryptionByDefault: {SSEAlgorithm: AES256}\n LifecycleConfiguration:\n Rules:\n - {Id: logs, Status: Enabled, Prefix: logs/, ExpirationInDays: 30}\n # The control instance's samples, a file a day (orca.system).\n - {Id: control, Status: Enabled, Prefix: logs/control/, ExpirationInDays: 14}\n # The lake's tables; anyone may read them when PublicData is true.\n # Account-level Block Public Access, if it's on, refuses the policy below.\n PublicBucket:\n Type: AWS::S3::Bucket\n Properties:\n BucketName: !Sub mlake-public-${AWS::AccountId}\n OwnershipControls:\n Rules: [{ObjectOwnership: BucketOwnerEnforced}]\n PublicAccessBlockConfiguration:\n BlockPublicAcls: true\n IgnorePublicAcls: true\n BlockPublicPolicy: !If [Public, false, true]\n RestrictPublicBuckets: !If [Public, false, true]\n BucketEncryption:\n ServerSideEncryptionConfiguration:\n - ServerSideEncryptionByDefault: {SSEAlgorithm: AES256}\n # Browsers, such as DuckDB in a page, read it too.\n CorsConfiguration:\n CorsRules:\n - {AllowedMethods: [GET, HEAD], AllowedOrigins: [\22*\22], AllowedHeaders: [\22*\22], MaxAge: 3600}\n PublicBucketPolicy:\n Type: AWS::S3::BucketPolicy\n Condition: Public\n Properties:\n Bucket: !Ref PublicBucket\n PolicyDocument:\n Statement:\n - Effect: Allow\n Principal: \22*\22\n Action: s3:GetObject\n Resource: !Sub ${PublicBucket.Arn}/*\n - Effect: Allow\n Principal: \22*\22\n Action: s3:ListBucket\n Resource: !GetAtt PublicBucket.Arn\n\n Repository:\n Type: AWS::ECR::Repository\n Properties:\n RepositoryName: orca\n ImageScanningConfiguration: {ScanOnPush: true}\n LifecyclePolicy:\n LifecyclePolicyText: |\n {\22rules\22: [{\22rulePriority\22: 1, \22description\22: \22keep the last 5\22,\n \22selection\22: {\22tagStatus\22: \22any\22, \22countType\22: \22imageCountMoreThan\22, \22countNumber\22: 5},\n \22action\22: {\22type\22: \22expire\22}}]}\n\n RunGroup:\n Type: AWS::EC2::SecurityGroup\n Properties:\n GroupDescription: orca run instances, which talk to each other only\n VpcId: !Ref Vpc\n RunGroupSelf:\n Type: AWS::EC2::SecurityGroupIngress\n Properties:\n GroupId: !Ref RunGroup\n SourceSecurityGroupId: !Ref RunGroup\n IpProtocol: tcp\n FromPort: 0\n ToPort: 65535\n # The scheduler reads a scaling run's demand from its head (dista's port).\n RunGroupHead:\n Type: AWS::EC2::SecurityGroupIngress\n Properties:\n GroupId: !Ref RunGroup\n SourceSecurityGroupId: !Ref ControlGroup\n IpProtocol: tcp\n FromPort: 6390\n ToPort: 6390\n ControlGroup:\n Type: AWS::EC2::SecurityGroup\n Properties:\n GroupDescription: orca control instance, public for HTTPS only\n VpcId: !Ref Vpc\n SecurityGroupIngress:\n - {IpProtocol: tcp, FromPort: 80, ToPort: 80, CidrIp: 0.0.0.0/0}\n - {IpProtocol: tcp, FromPort: 443, ToPort: 443, CidrIp: 0.0.0.0/0}\n - {IpProtocol: tcp, FromPort: 5432, ToPort: 5432, SourceSecurityGroupId: !Ref RunGroup}\n\n RunRole:\n Type: AWS::IAM::Role\n Properties:\n AssumeRolePolicyDocument:\n Statement: [{Effect: Allow, Principal: {Service: ec2.amazonaws.com}, Action: sts:AssumeRole}]\n ManagedPolicyArns: [arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore]\n Policies:\n # Runs' secrets (orca register --secret), only those under orca/.\n - PolicyName: secrets\n PolicyDocument:\n Statement:\n - Effect: Allow\n Action: secretsmanager:GetSecretValue\n Resource: !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:orca/*\n - PolicyName: bucket-and-image\n PolicyDocument:\n Statement:\n - Effect: Allow\n Action: [s3:GetObject, s3:PutObject, s3:DeleteObject]\n Resource: [!Sub \22${Bucket.Arn}/*\22, !Sub \22${PublicBucket.Arn}/*\22]\n # The control instance's backups (orca aws replace-control)\n # hold the database and Caddy's keys: no run reads them.\n - Effect: Deny\n Action: [s3:GetObject, s3:PutObject, s3:DeleteObject]\n Resource: !Sub \22${Bucket.Arn}/backups/*\22\n - Effect: Allow\n Action: s3:ListBucket\n Resource: [!GetAtt Bucket.Arn, !GetAtt PublicBucket.Arn]\n - Effect: Allow\n Action: [ecr:BatchGetImage, ecr:GetDownloadUrlForLayer]\n Resource: !GetAtt Repository.Arn\n - Effect: Allow\n Action: ecr:GetAuthorizationToken\n Resource: \22*\22\n RunProfile:\n Type: AWS::IAM::InstanceProfile\n Properties: {Roles: [!Ref RunRole]}\n\n ControlRole:\n Type: AWS::IAM::Role\n Properties:\n AssumeRolePolicyDocument:\n Statement: [{Effect: Allow, Principal: {Service: ec2.amazonaws.com}, Action: sts:AssumeRole}]\n ManagedPolicyArns: [arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore]\n Policies:\n # The secrets of runs placed on this instance (orca register\n # --placement control), which the scheduler reads as it starts them.\n - PolicyName: secrets\n PolicyDocument:\n Statement:\n - Effect: Allow\n Action: secretsmanager:GetSecretValue\n Resource: !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:orca/*\n - PolicyName: run-instances\n PolicyDocument:\n Statement:\n # Only instances tagged for orca, of the allowed types, from\n # the stack's AMI, in its subnet and group, with its profile.\n - Effect: Allow\n Action: ec2:RunInstances\n Resource: !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:instance/*\n Condition:\n StringEquals: {ec2:InstanceType: !Ref AllowedRunTypes}\n \22Null\22: {\22aws:RequestTag/orca.run\22: \22false\22}\n - Effect: Allow\n Action: ec2:RunInstances\n Resource:\n - !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:volume/*\n - !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*\n - !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:subnet/${Subnet}\n - !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${RunGroup}\n - !Sub arn:aws:ec2:${AWS::Region}::image/*\n - !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:spot-instances-request/*\n - Effect: Allow\n Action: ec2:CreateTags\n Resource: !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:*/*\n Condition:\n StringEquals: {ec2:CreateAction: RunInstances}\n - Effect: Allow\n Action: ec2:TerminateInstances\n Resource: !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:instance/*\n Condition:\n \22Null\22: {\22aws:ResourceTag/orca.run\22: \22false\22}\n # Its instances, and the boot script it runs (ControlBoot).\n - Effect: Allow\n Action: [ec2:DescribeInstances, ec2:DescribeLaunchTemplateVersions]\n Resource: \22*\22\n - Effect: Allow\n Action: iam:PassRole\n Resource: !GetAtt RunRole.Arn\n - PolicyName: bucket-and-image\n PolicyDocument:\n Statement:\n - Effect: Allow\n Action: [s3:GetObject, s3:PutObject, s3:DeleteObject]\n Resource: [!Sub \22${Bucket.Arn}/*\22, !Sub \22${PublicBucket.Arn}/*\22]\n - Effect: Allow\n Action: s3:ListBucket\n Resource: [!GetAtt Bucket.Arn, !GetAtt PublicBucket.Arn]\n - Effect: Allow\n Action: [ecr:BatchGetImage, ecr:GetDownloadUrlForLayer]\n Resource: !GetAtt Repository.Arn\n - Effect: Allow\n Action: ecr:GetAuthorizationToken\n Resource: \22*\22\n ControlProfile:\n Type: AWS::IAM::InstanceProfile\n Properties: {Roles: [!Ref ControlRole]}\n\n ControlAddress:\n Type: AWS::EC2::EIP\n Properties: {Domain: vpc}\n # The control instance's boot script, kept as the user data of a launch\n # template that launches nothing: each control instance's own user data\n # runs the latest version of it, so changing it (or a parameter it uses)\n # stops no running instance. A new instance gets it: orca aws\n # replace-control, which boots one in the other slot.\n ControlBoot:\n Type: AWS::EC2::LaunchTemplate\n Properties:\n LaunchTemplateData:\n UserData:\n Fn::Base64: !Sub\n - |\n #!/bin/bash\n # orca-boot SLOT: orca's control instance, in slot blue or\n # green: Postgres, the scheduler, the UI and Caddy, as\n # containers that restart with the instance.\n set -eux\n slot=$1\n # Docker without ECS: this AMI's agent would look for a cluster.\n systemctl disable --now ecs || true\n systemctl enable --now docker\n command -v aws || dnf install -y awscli\n token=$(curl -sX PUT http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 300')\n md() { curl -s -H \22X-aws-ec2-metadata-token: $token\22 http://169.254.169.254/latest/meta-data/$1; }\n ip=$(md local-ipv4)\n # The Elastic IP's sslip.io name works at once; the hostname once\n # its A record points here. Caddy gets a certificate for each.\n sslip=\22$(echo ${PublicIp} | tr . -).sslip.io\22\n hosts=\22$sslip\22\n [ -z \22${Hostname}\22 ] || hosts=\22${Hostname}, $sslip\22\n registry=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com\n image=$registry/orca:${ImageTag}\n login=\22command -v aws >/dev/null || dnf install -y awscli; systemctl disable --now ecs || true; aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin $registry\22\n # The image is pushed after the stack is made, so wait for it.\n until sh -c \22$login\22 && docker pull $image; do sleep 30; done\n docker network create orca || true\n mkdir -p /var/lib/orca/pg /var/lib/orca/caddy /var/lib/orca/stage\n docker run -d --name postgres --restart always --network orca -p 5432:5432 \\\n -e POSTGRES_USER=orca -e POSTGRES_DB=orca -e POSTGRES_PASSWORD='${DbPassword}' \\\n -v /var/lib/orca/pg:/var/lib/postgresql/data postgres:17-bookworm -c synchronous_commit=on\n # Over TCP: the server that initialises the database listens on\n # its socket only, and restarts once it's done.\n until docker exec postgres pg_isready -h 127.0.0.1 -U orca -d orca; do sleep 2; done\n # A replacement (orca aws replace-control) points this slot at\n # the backup of the instance it replaces: its database, image\n # tag, Caddy's certificates and apps come back before anything\n # starts on them.\n backups=s3://${Bucket}/backups/${AWS::StackName}\n if aws s3 cp --region ${AWS::Region} $backups/restore-$slot /var/lib/orca/restore-from; then\n from=$(cat /var/lib/orca/restore-from)\n aws s3 cp --region ${AWS::Region} \22$from.dump\22 /var/lib/orca/restore.dump\n docker exec -i postgres pg_restore -U orca -d orca --no-owner --exit-on-error < /var/lib/orca/restore.dump\n rm -f /var/lib/orca/restore.dump\n aws s3 cp --region ${AWS::Region} \22$from.tag\22 /var/lib/orca/tag\n aws s3 cp --region ${AWS::Region} \22$from.caddy.tgz\22 - | tar -xz -C /var/lib/orca/caddy\n aws s3 cp --region ${AWS::Region} \22$from.apps\22 /var/lib/orca/apps\n echo \22$from\22 > /var/lib/orca/restored\n fi\n [ -f /var/lib/orca/tag ] || echo ${ImageTag} > /var/lib/orca/tag\n cat > /etc/orca.env << EOF\n ORCA_DATABASE_URL=postgresql://orca:${DbPassword}@postgres:5432/orca\n ORCA_EC2_DATABASE_URL=postgresql://orca:${DbPassword}@$ip:5432/orca\n ORCA_RUNNER=ec2\n AWS_REGION=${AWS::Region}\n ORCA_EC2_AMI=${Ami}\n ORCA_EC2_SUBNET=${Subnet}\n ORCA_EC2_SECURITY_GROUP=${RunGroup}\n ORCA_EC2_PROFILE=${RunProfile}\n ORCA_EC2_PREAMBLE=$login\n ORCA_WAREHOUSE=s3://${PublicBucket}/warehouse\n ORCA_LOGS=s3://${Bucket}/logs\n ORCA_PUBLIC_WAREHOUSE=s3://${PublicBucket}/warehouse\n ORCA_RUN_ENV=ORCA_PUBLIC_WAREHOUSE\n ORCA_LOG_STAGE=/var/lib/orca/stage\n ORCA_CONTROL_SAMPLES=/var/lib/orca/stage/control.jsonl\n ORCA_CONTROL_TYPE=${ControlInstanceType}\n ORCA_UI_ADDR=0.0.0.0:8270\n ORCA_UI_LOGIN=required\n ORCA_UI_SECURE_COOKIES=1\n ORCA_UI_TRUST_PROXY=1\n EOF\n # The stack's own settings (ControlEnv; orca aws up --env K=V),\n # each in place of the line above with its key.\n printf '%s;' '${ControlEnv}' | tr ';' '\\n' | while IFS= read -r kv; do\n [ -n \22$kv\22 ] || continue\n k=$(printf '%s' \22$kv\22 | cut -d= -f1)\n grep -v \22^$k=\22 /etc/orca.env > /etc/orca.env.new || true\n printf '%s\\n' \22$kv\22 >> /etc/orca.env.new\n mv /etc/orca.env.new /etc/orca.env\n done\n chmod 600 /etc/orca.env\n # orca aws register and orca aws app run these over SSM.\n cat > /usr/local/bin/orca-refresh << 'EOS'\n #!/bin/bash\n # orca-refresh [TAG]: pull the orca image at TAG (else the last\n # one) and run the scheduler and the UI from it again.\n set -eu\n repo=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/orca\n [ $# -gt 0 ] && echo \22$1\22 > /var/lib/orca/tag\n tag=$(cat /var/lib/orca/tag)\n aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin \22$(echo $repo | cut -d/ -f1)\22 > /dev/null\n docker pull -q \22$repo:$tag\22\n for role in scheduler ui; do\n docker rm -f $role > /dev/null 2>&1 || true\n docker run -d --name $role --restart always --network orca --env-file /etc/orca.env \\\n -v /var/lib/orca/stage:/var/lib/orca/stage \22$repo:$tag\22 $role\n done\n # This instance's own samples, for the fleet and system pages: the\n # host's processes and network, the last hour of lines kept in a\n # file the scheduler reads. The host's cgroup files and Docker's\n # container configs, read-only, give each container's memory by\n # name without the Docker socket.\n docker rm -f sampler > /dev/null 2>&1 || true\n docker run -d --name sampler --restart always --pid host --network host \\\n -v /var/lib/orca/stage:/var/lib/orca/stage \\\n -v /sys/fs/cgroup:/host/cgroup:ro -v /var/lib/docker/containers:/host/containers:ro \22$repo:$tag\22 \\\n sample --out /var/lib/orca/stage/control.jsonl --dirs /var/lib/orca/stage --keep 720 \\\n --cgroup /host/cgroup --names /host/containers\n EOS\n cat > /usr/local/bin/orca-app << 'EOS'\n #!/bin/bash\n # orca-app NAME HOST [PORT]: run the image's /opt/orca/bin/NAME as\n # an app beside the UI, at https://HOST through Caddy, with the\n # region, the warehouse and PORT (default 8080). It's recorded in\n # /var/lib/orca/apps, which a replacement instance runs again.\n set -eu\n name=$1\n host=$2\n port=8080\n [ $# -gt 2 ] && port=$3\n repo=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/orca\n tag=$(cat /var/lib/orca/tag)\n aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin \22$(echo $repo | cut -d/ -f1)\22 > /dev/null\n docker pull -q \22$repo:$tag\22\n docker rm -f \22$name\22 > /dev/null 2>&1 || true\n # An app reads the lake, not orca's database: it gets no password.\n grep -E '^(AWS_REGION|ORCA_WAREHOUSE|ORCA_PUBLIC_WAREHOUSE)=' /etc/orca.env > /var/lib/orca/app.env\n docker run -d --name \22$name\22 --restart always --network orca --env-file /var/lib/orca/app.env \\\n -e PORT=$port --entrypoint /opt/orca/bin/$name \22$repo:$tag\22\n { grep -v \22^$name \22 /var/lib/orca/apps 2> /dev/null || true; echo \22$name $host $port\22; } > /var/lib/orca/apps.new\n mv /var/lib/orca/apps.new /var/lib/orca/apps\n grep -q \22^$host {\22 /var/lib/orca/Caddyfile \\\n || printf '%s {\\n encode zstd gzip\\n reverse_proxy %s:%s\\n}\\n' \22$host\22 \22$name\22 \22$port\22 >> /var/lib/orca/Caddyfile\n # Caddy may still be starting.\n for _ in $(seq 30); do\n docker exec caddy caddy reload --config /etc/caddy/Caddyfile && exit 0\n sleep 2\n done\n exit 1\n EOS\n chmod +x /usr/local/bin/orca-refresh /usr/local/bin/orca-app\n orca-refresh\n cat > /var/lib/orca/Caddyfile << EOF\n $hosts {\n encode zstd gzip\n reverse_proxy ui:8270\n }\n EOF\n # The console, once a container named console runs on the network.\n [ -z \22${ConsoleHostname}\22 ] || printf '%s {\\n encode zstd gzip\\n reverse_proxy console:8080\\n}\\n' \22${ConsoleHostname}\22 >> /var/lib/orca/Caddyfile\n docker run -d --name caddy --restart always --network orca -p 80:80 -p 443:443 \\\n -v /var/lib/orca/Caddyfile:/etc/caddy/Caddyfile -v /var/lib/orca/caddy:/data caddy:2.10\n # The apps the replaced instance ran.\n if [ -f /var/lib/orca/apps ]; then\n cp /var/lib/orca/apps /var/lib/orca/apps.boot\n while read -r name host port; do orca-app \22$name\22 \22$host\22 \22$port\22; done < /var/lib/orca/apps.boot\n fi\n echo \22orca ui at https://$sslip and https://${Hostname}\22\n date -u +%FT%TZ > /var/lib/orca/booted\n - {PublicIp: !Ref ControlAddress}\n # Two slots for the control instance, blue (Control, the stack's first)\n # and green; the Elastic IP is on the active one. orca aws\n # replace-control turns the other slot on with a fresh instance, moves\n # ActiveControl to it once it's healthy, and turns the old slot off at the\n # end of the rollback window. Each slot's AMI is pinned (BlueAmi,\n # GreenAmi): the Ami parameter resolves again on every update, and a new\n # image id would replace the running instance, its database with it.\n Control:\n Type: AWS::EC2::Instance\n Condition: BlueOn\n DependsOn: DefaultRoute\n Properties:\n ImageId: !If [BlueAmiPinned, !Ref BlueAmi, !Ref Ami]\n InstanceType: !Ref ControlInstanceType\n SubnetId: !Ref Subnet\n SecurityGroupIds: [!Ref ControlGroup]\n IamInstanceProfile: !Ref ControlProfile\n MetadataOptions: {HttpTokens: required, HttpPutResponseHopLimit: 2}\n BlockDeviceMappings:\n - DeviceName: /dev/xvda\n Ebs: {VolumeSize: 40, VolumeType: gp3, Encrypted: true}\n Tags: [{Key: Name, Value: orca-control}, {Key: orca.slot, Value: blue}]\n UserData:\n Fn::Base64: !Sub |\n #!/bin/bash\n # orca's control instance, slot blue: the latest version of the\n # stack's boot script (ControlBoot).\n set -eux\n command -v aws || dnf install -y awscli\n until aws ec2 describe-launch-template-versions --region ${AWS::Region} --launch-template-id ${ControlBoot} \\\n --versions '$Latest' --query 'LaunchTemplateVersions[0].LaunchTemplateData.UserData' --output text > /root/orca-boot.b64; do sleep 10; done\n base64 -d /root/orca-boot.b64 > /usr/local/bin/orca-boot\n chmod 700 /usr/local/bin/orca-boot\n exec /usr/local/bin/orca-boot blue\n ControlGreen:\n Type: AWS::EC2::Instance\n Condition: GreenOn\n DependsOn: DefaultRoute\n Properties:\n ImageId: !If [GreenAmiPinned, !Ref GreenAmi, !Ref Ami]\n InstanceType: !Ref ControlInstanceType\n SubnetId: !Ref Subnet\n SecurityGroupIds: [!Ref ControlGroup]\n IamInstanceProfile: !Ref ControlProfile\n MetadataOptions: {HttpTokens: required, HttpPutResponseHopLimit: 2}\n BlockDeviceMappings:\n - DeviceName: /dev/xvda\n Ebs: {VolumeSize: 40, VolumeType: gp3, Encrypted: true}\n Tags: [{Key: Name, Value: orca-control}, {Key: orca.slot, Value: green}]\n UserData:\n Fn::Base64: !Sub |\n #!/bin/bash\n # orca's control instance, slot green: the latest version of the\n # stack's boot script (ControlBoot).\n set -eux\n command -v aws || dnf install -y awscli\n until aws ec2 describe-launch-template-versions --region ${AWS::Region} --launch-template-id ${ControlBoot} \\\n --versions '$Latest' --query 'LaunchTemplateVersions[0].LaunchTemplateData.UserData' --output text > /root/orca-boot.b64; do sleep 10; done\n base64 -d /root/orca-boot.b64 > /usr/local/bin/orca-boot\n chmod 700 /usr/local/bin/orca-boot\n exec /usr/local/bin/orca-boot green\n # Its own resource, so moving the address is a change to it alone.\n ControlAddressAssociation:\n Type: AWS::EC2::EIPAssociation\n Properties:\n InstanceId: !If [GreenActive, !Ref ControlGreen, !Ref Control]\n AllocationId: !GetAtt ControlAddress.AllocationId\n\n Budget:\n Type: AWS::Budgets::Budget\n Properties:\n Budget:\n BudgetName: orca\n BudgetType: COST\n TimeUnit: MONTHLY\n BudgetLimit: {Amount: !Ref BudgetDollars, Unit: USD}\n NotificationsWithSubscribers:\n - Notification: {NotificationType: ACTUAL, ComparisonOperator: GREATER_THAN, Threshold: 80}\n Subscribers: [{SubscriptionType: EMAIL, Address: !Ref BudgetEmail}]\n - Notification: {NotificationType: FORECASTED, ComparisonOperator: GREATER_THAN, Threshold: 100}\n Subscribers: [{SubscriptionType: EMAIL, Address: !Ref BudgetEmail}]\n\nOutputs:\n SslipUrl:\n Value: !Sub\n - https://${Dashed}.sslip.io\n - Dashed: !Join [\22-\22, !Split [\22.\22, !Ref ControlAddress]]\n Repository:\n Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/orca\n Bucket:\n Value: !Ref Bucket\n PublicBucket:\n Description: The lake's bucket; with PublicData, read it unsigned, e.g. aws s3 ls --no-sign-request s3://<it>/\n Value: !Ref PublicBucket\n ControlInstance:\n Description: The active control instance, which has the Elastic IP\n Value: !If [GreenActive, !Ref ControlGreen, !Ref Control]\n ActiveControl:\n Value: !Ref ActiveControl\n BlueInstance:\n Condition: BlueOn\n Value: !Ref Control\n GreenInstance:\n Condition: GreenOn\n Value: !Ref ControlGreen\n AddressAllocation:\n Value: !GetAtt ControlAddress.AllocationId\n ConsoleHostname:\n Value: !Ref ConsoleHostname\n PublicAddress:\n Value: !Ref ControlAddress\n")orca.aws_assets
The stack template and the project Dockerfile orca aws writes out: deploy/aws/orca.yaml and deploy/project.Dockerfile, embedded by scripts/embed-aws.py. Edit those, then run it; pixi run test-aws-assets fails while this is out of date.
Aliases
comptime PROJECT_DOCKERFILE = StringSpan("# The image `orca aws push` builds for a project that added orca with\n# `pixi shelf add orca-mojo`: its own `orca` (bin/scheduler.mojo, the two\n# lines calling orca.command), its apps (every other bin/*.mojo, at\n# /opt/orca/bin/<name>, which `orca aws app` runs; `orca_maintain`, the\n# snapshot expiry `orca expire` runs, is made for a project whose pipelines\n# use orca.tables and that has no bin/orca_maintain.mojo of its own) and its\n# pipelines (every\n# pipelines/*.mojo, at /opt/orca/pipelines/<name>), with `src/` on the\n# import path if the project has one. A project with a Dockerfile.orca of\n# its own builds that instead.\n#\n# ORCA_SOURCE and ORCA_COMMIT say what the pipelines were built from, which\n# each run records and orca ui links.\n#\n# Everything is built with line tables (-g1) and the image has\n# llvm-symbolizer, so a crash's stack dump in a run's driver.log names each\n# frame's function, file and line (orca.crash). The line tables, compressed,\n# make a binary about half as big again (mojo-census's gharchive: 4.2 MB,\n# 6.5 MB with them, 25 MB uncompressed) and leave its speed as it was.\n#\n# The binaries are built for MOJO_TARGET_CPU, not for the CPU of the machine\n# building them (`mojo build` defaults to that). GitHub's arm64 runners are\n# Neoverse N2, which has SVE2; Hetzner's CAX servers are Neoverse N1, which\n# has not, so a binary built there dies on the server with SIGILL (exit\n# 132). N1 code runs on N2, V1, V2 and Graviton. Build for something else\n# with --build-arg MOJO_TARGET_CPU=<cpu>; this is for linux/arm64 images.\nFROM ghcr.io/prefix-dev/pixi:0.78.0 AS build\nARG MOJO_TARGET_CPU=neoverse-n1\nARG ORCA_SOURCE=\22\22\nARG ORCA_COMMIT=\22\22\nRUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev git ca-certificates \\\n && rm -rf /var/lib/apt/lists/*\nWORKDIR /src/project\nCOPY pixi.toml pixi.lock ./\nRUN pixi install --locked\nCOPY . .\nRUN mkdir -p /out/bin /out/pipelines \\\n && pixi run sh -c 'inc=\22\22; [ -d src ] && inc=\22-I src\22; \\\n [ -f bin/scheduler.mojo ] || { echo \22orca aws push: no bin/scheduler.mojo calling orca.command\22; exit 1; }; \\\n for f in bin/*.mojo; do n=$(basename \22$f\22 .mojo); [ \22$n\22 = scheduler ] && n=orca; \\\n mojo build -g1 --target-cpu \22$MOJO_TARGET_CPU\22 $inc \22$f\22 -o \22/out/bin/$n\22 || exit 1; done; \\\n if [ ! -f bin/orca_maintain.mojo ] && grep -rqs \22orca.tables\22 pipelines src; then \\\n printf \22from orca.maintain import maintain_main\\n\\n\\ndef main() raises:\\n maintain_main()\\n\22 > /tmp/orca_maintain.mojo; \\\n mojo build -g1 --target-cpu \22$MOJO_TARGET_CPU\22 $inc /tmp/orca_maintain.mojo -o /out/bin/orca_maintain || exit 1; fi; \\\n for p in pipelines/*.mojo; do [ -f \22$p\22 ] || continue; n=$(basename \22$p\22 .mojo); \\\n mojo build -g1 --target-cpu \22$MOJO_TARGET_CPU\22 $inc -D ORCA_SOURCE='\22${ORCA_SOURCE}\22' -D ORCA_COMMIT='\22${ORCA_COMMIT}\22' \\\n \22$p\22 -o \22/out/pipelines/$n\22 || exit 1; done; \\\n if ls /out/pipelines/* > /dev/null 2>&1; then /out/bin/orca locate /out/pipelines/* --root .; fi'\n# The line tables compressed, which llvm-symbolizer reads as they are.\nRUN for b in /out/bin/* /out/pipelines/*; do [ -x \22$b\22 ] || continue; \\\n objcopy --compress-debug-sections=zlib \22$b\22 || exit 1; done\n# Only the pixi environment's libraries the binaries link or dlopen: libpq\n# (postgres), flare's shims (orca ui, apps), and every tin's shim, such as\n# objectstore's and the codecs' (zstd, for Iceberg writes); and\n# llvm-symbolizer, which Mojo's runtime runs to name a crash's frames, with\n# what it links.\nRUN env=/src/project/.pixi/envs/default && mkdir -p /out/lib /out/libexec \\\n && cp -L $env/bin/llvm-symbolizer /out/libexec/ \\\n && loaded=$(ls $env/lib/libpq.so* $env/lib/libflare_*.so $env/lib/lib*mojo*.so 2>/dev/null) \\\n && ldd /out/bin/* /out/pipelines/* /out/libexec/* $loaded 2> /dev/null | awk -v e=\22$env\22 '$3 ~ \22^\22e {print $3}' \\\n | sort -u | xargs -r cp -L -t /out/lib \\\n && cp -L -t /out/lib $loaded\n\nFROM debian:bookworm-slim\nRUN apt-get update && apt-get install -y --no-install-recommends tzdata ca-certificates \\\n && rm -rf /var/lib/apt/lists/*\nCOPY --from=build /out/lib /opt/orca/lib\nCOPY --from=build /out/bin /opt/orca/bin\nCOPY --from=build /out/pipelines /opt/orca/pipelines\nCOPY --from=build /out/libexec /opt/orca/libexec\nENV CONDA_PREFIX=/opt/orca LD_LIBRARY_PATH=/opt/orca/lib PATH=/opt/orca/bin:/opt/orca/pipelines:/usr/bin:/bin \\\n LLVM_SYMBOLIZER_PATH=/opt/orca/libexec/llvm-symbolizer\nENTRYPOINT [\22/opt/orca/bin/orca\22]\n")