Deploy ClickHouse on AWS EKS

# Replace 0000000000 in the Resource ARN below with the ClickHouse Private source ECR account ID
AWSTemplateFormatVersion: 2010-09-09
Description: This CloudFormation template creates the ClickHouseAirgapECRPullRole.

Resources:
  ClickHouseAirgapECRPullRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: ClickHouseAirgapECRPullRole
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              AWS: !Sub "arn:aws:iam::${AWS::AccountId}:root"
            Action:
              - sts:AssumeRole
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: ClickHouseAirgapECRPullPolicy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - ecr:GetDownloadUrlForLayer
                  - ecr:BatchGetImage
                  - ecr:BatchCheckLayerAvailability
                  - ecr:DescribeImages
                  - ecr:ListImages
                Resource:
                  - arn:aws:ecr:us-east-1:0000000000:repository/*
              - Effect: Allow
                Action:
                  - ecr:GetAuthorizationToken
                Resource:
                  - "*"

Outputs:
  RoleArn:
    Description: IAM Role ARN for pulling images from all ECR repositories
    Value: !GetAtt ClickHouseAirgapECRPullRole.Arn

SOURCE_ECR_ACCOUNT_ID=<account-id-from-onboarding>
SOURCE_REGION=us-east-1
SOURCE_ECR_REPO=$SOURCE_ECR_ACCOUNT_ID.dkr.ecr.$SOURCE_REGION.amazonaws.com

TARGET_REGION=us-west-2
TARGET_ECR_REPO=0000000000.dkr.ecr.$TARGET_REGION.amazonaws.com

# log into our (authentication process may differ for customer)
aws ecr get-login-password --region $SOURCE_REGION | skopeo login --username AWS --password-stdin $SOURCE_ECR_REPO

# log into the target AWS repo (authentication process may differ for customer)
aws ecr get-login-password --region $TARGET_REGION | skopeo login --username AWS --password-stdin $TARGET_ECR_REPO

# copy each image to target ECR, be sure to include the --all flag
skopeo copy --all docker://$SOURCE_ECR_REPO/clickhouse-server:26.2.1.525 docker://$TARGET_ECR_REPO/clickhouse-server:26.2.1.525
skopeo copy --all docker://$SOURCE_ECR_REPO/clickhouse-keeper:26.2.1.258 docker://$TARGET_ECR_REPO/clickhouse-keeper:26.2.1.258
skopeo copy --all docker://$SOURCE_ECR_REPO/clickhouse-operator:main-1.20930.1 docker://$TARGET_ECR_REPO/clickhouse-operator:main-1.20930.1
skopeo copy --all docker://$SOURCE_ECR_REPO/helm/clickhouse-operator-helm:1.20930.1 docker://$TARGET_ECR_REPO/helm/clickhouse-operator-helm:1.20930.1
skopeo copy --all docker://$SOURCE_ECR_REPO/helm/onprem-clickhouse-cluster:1.8.7 docker://$TARGET_ECR_REPO/helm/onprem-clickhouse-cluster:1.8.7

REGION=us-west-2
EKS_CLUSTER_NAME=eks-clickhouse

aws eks update-kubeconfig --region $REGION --name=$EKS_CLUSTER_NAME

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "Federated": "arn:aws:iam::0000000000:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXX"
            },
            "Action": "sts:AssumeRoleWithWebIdentity",
            "Condition": {
                "StringEquals": {
                    "oidc.eks.us-west-2.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXX:aud": "sts.amazonaws.com",
"oidc.eks.us-west-2.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:$NAMESPACE:$K8S_SERVICE_ACCOUNT"
                }
            }
        }
    ]
}

kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml

# update ECR_HOST as needed
ECR_HOST=0000000000.dkr.ecr.us-west-2.amazonaws.com
CHART_VERSION=1.8.7

helm install clickhouse-prerequisites \
    oci://$ECR_HOST/helm/onprem-clickhouse-cluster \
    --version=$CHART_VERSION \
    -n default \
    --set-json="storageClass.create=true" \
    --set-json="createCluster=false" \
    --set-json="serviceAccount.create=false"

# update role name as needed
AWS_EBS_CSI_DRIVER_IAM_ROLE_ARN=arn:aws:iam::0000000000:role/ClickHouse_EksEbsCsiDriverRole

helm upgrade --install aws-ebs-csi-driver \
--namespace kube-system \
aws-ebs-csi-driver/aws-ebs-csi-driver \
--set-json="controller.serviceAccount.annotations={\"eks.amazonaws.com/role-arn\":\"$AWS_EBS_CSI_DRIVER_IAM_ROLE_ARN\"}"

ECR_HOST=0000000000.dkr.ecr.us-west-2.amazonaws.com
REGION=us-west-2

 aws ecr get-login-password \
  --region $REGION | helm registry login \
  --username AWS \
  --password-stdin $ECR_HOST

# update ECR_HOST as needed
ECR_HOST=0000000000.dkr.ecr.us-west-2.amazonaws.com

# chart version -- the operator's Helm chart tag (eg 1.20930.1, not main-1.20930.1)
OPERATOR_VERSION=1.20930.1

# image tag -- the operator build itself (note the main- prefix). Always set this
# explicitly so the operator version is pinned for this install.
OPERATOR_IMAGE_TAG=main-1.20930.1

# set AZs as determined by VPC subnets
AZ_LIST='["us-west-2a","us-west-2b","us-west-2c"]'

helm install clickhouse-operator \
   oci://$ECR_HOST/helm/clickhouse-operator-helm \
   --version=$OPERATOR_VERSION \
   --create-namespace \
   -n clickhouse-operator-system \
   --set-json="image.repository=\"$ECR_HOST/clickhouse-operator\"" \
   --set-json="image.tag=\"$OPERATOR_IMAGE_TAG\"" \
   --set-json='cilium.enabled=false' \
   --set-json='idleScalerEnabled=false' \
   --set-json='webhooks.enabled=false' \
   --set-json="operator.availabilityZones=$AZ_LIST"

# this will be the `default` user's password
PASSWORD='My super secret p@$$w0rd'
if command -v sha256sum &> /dev/null; then
  HASHED_PASSWORD=$(echo -n "$PASSWORD" | sha256sum | awk '{printf $1}' | base64 | tr -d '\n')
else
  HASHED_PASSWORD=$(echo -n "$PASSWORD" | shasum -a 256 | awk '{printf $1}' | base64 | tr -d '\n')
fi

# update values below as needed,
# the ecr host should be the one in your account
# the s3 bucket should be the one created earlier
CLUSTER_NAME=default-xx-01
ECR_HOST=0000000000.dkr.ecr.us-west-2.amazonaws.com
SA_IAM_ROLE_ARN=arn:aws:iam::0000000000:role/CH-S3-${CLUSTER_NAME}-uw2-00-Role
S3_BUCKET_NAME=my-clickhouse-data

# S3 bucket region -- must match the bucket. The chart defaults to us-west-2; set
# this (and the derived endpoint below) whenever the bucket lives elsewhere.
S3_BUCKET_REGION=us-west-2

# s3 key prefix can use any UUID value, but must be unique for all clusters storing data in the bucket specified above
S3_KEY_PREFIX=ch-s3-$(uuidgen | tr '[:upper:]' '[:lower:]')

# these should match the clickhouseGroup label value added to the node group
SERVER_NG_GROUP=server-arm64
KEEPER_NG_GROUP=keeper-arm64

# these values should change depending on selected instance sizes
# be sure to take daemon set requirements into account when setting CPU and MEMORY values
SERVER_CPU=64
SERVER_MEMORY=256Gi
# Keeper: high-load profile; for moderate load 2-4 CPU / 8Gi also works (see the infrastructure reference)
KEEPER_CPU=4
KEEPER_MEMORY=16Gi

CHART_VERSION=1.8.7

# bytesPerGiRAM is a scaling factor used to automatically calculate the disk cache size.
# As a general rule, set cache size to 70-80% of the allocatable SSD disk, accounting for DaemonSets and Cloud Provider reserved disk space.
# When you set bytesPerGiRAM in the Helm chart, the following formula is used during pod startup:
#
# CONFIG_DISK_CACHE_SIZE = bytesPerGiRAM * pod_memory_limit
#
# For example:
# bytesPerGiRAM = 11.3Gi
# pod_memory_limit = 250Gi
# CONFIG_DISK_CACHE_SIZE = 11.3 * 250 = 2825Gi
CACHE_BYTES_PER_GI_RAM=11Gi


helm install $CLUSTER_NAME \
    oci://$ECR_HOST/helm/onprem-clickhouse-cluster \
    --version=$CHART_VERSION \
    -n ns-$CLUSTER_NAME \
    --create-namespace \
    --set-json="account.hashedPassword=\"$HASHED_PASSWORD\"" \
    --set-json="server.image.repository=\"$ECR_HOST/clickhouse-server\"" \
    --set-json="server.storage.s3.bucketName=\"$S3_BUCKET_NAME\"" \
    --set-json="server.storage.s3.region=\"$S3_BUCKET_REGION\"" \
    --set-json="server.storage.s3.endpoint=\"https://s3.$S3_BUCKET_REGION.amazonaws.com\"" \
    --set-json="server.storage.s3.keyPrefix=\"$S3_KEY_PREFIX\"" \
    --set-json='server.arm64=true' \
    --set-json="server.podPolicy.nodeSelector.clickhouseGroup=\"$SERVER_NG_GROUP\"" \
    --set-json="keeper.image.repository=\"$ECR_HOST/clickhouse-keeper\"" \
    --set-json='keeper.arm64=true' \
    --set-json="keeper.podPolicy.nodeSelector.clickhouseGroup=\"$KEEPER_NG_GROUP\"" \
    --set-json="serviceAccount.annotations={\"eks.amazonaws.com/role-arn\":\"$SA_IAM_ROLE_ARN\"}" \
    --set-json='server.tolerations=[{"effect":"NoSchedule","key":"clickhouse.com/do-not-schedule","operator":"Exists"}]' \
    --set-json='keeper.tolerations=[{"effect":"NoSchedule","key":"clickhouse.com/do-not-schedule","operator":"Exists"}]' \
    --set-json="server.ssdCacheConfiguration.bytesPerGiRAM=\"$CACHE_BYTES_PER_GI_RAM\"" \
    --set-json="server.podPolicy.resources.limits.cpu=\"$SERVER_CPU\"" \
    --set-json="server.podPolicy.resources.limits.memory=\"$SERVER_MEMORY\"" \
    --set-json="server.podPolicy.resources.requests.cpu=\"$SERVER_CPU\"" \
    --set-json="server.podPolicy.resources.requests.memory=\"$SERVER_MEMORY\"" \
    --set-json="keeper.podPolicy.resources.limits.cpu=\"$KEEPER_CPU\"" \
    --set-json="keeper.podPolicy.resources.limits.memory=\"$KEEPER_MEMORY\"" \
    --set-json="keeper.podPolicy.resources.requests.cpu=\"$KEEPER_CPU\"" \
    --set-json="keeper.podPolicy.resources.requests.memory=\"$KEEPER_MEMORY\"" \
    --set-json="keeper.storage.resources.requests=\"50Gi\"" \
    # --set-json='server.ssdCacheConfiguration.isOnEmptyDir=true'  # Uncomment if using Bottlerocket (see Step 5)

kubectl krew install preflight

skopeo copy --all docker://$SOURCE_ECR_REPO/helm/preflight-check:1.38.4 docker://$TARGET_ECR_REPO/helm/preflight-check:1.38.4

ECR_HOST=0000000000.dkr.ecr.us-west-2.amazonaws.com
CHART_VERSION=1.38.4
CLUSTER_NAME=default-xx-01

helm template clickhouse-preflight \
    oci://$ECR_HOST/helm/preflight-check \
    --version=$CHART_VERSION \
    --set preflight.cloud=aws \
    --set preflight.clickhouseClusterName=$CLUSTER_NAME | \
kubectl preflight -

kubectl port-forward svc/c-default-xx-01-server-any 9000:9000 -n ns-default-xx-01

clickhouse client --host localhost --port 9000 --password $PASSWORD

SELECT 1;

   ┌─1─┐
1. │ 1 │
   └───┘
1 row in set. Elapsed: 0.001 sec.