Deploy ClickHouse with FIPS Compliance (Government)

kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml

ECR_HOST=0000000000.dkr-ecr-fips.us-west-2.on.aws
CHART_VERSION=1.8.7

helm install clickhouse-prerequisites \
    oci://$ECR_HOST/helm/onprem-clickhouse-cluster \
    --version=$CHART_VERSION \
    -n default \
    --set-json="storageClass.create=true" \
    --set-json="createCluster=false" \
    --set-json="serviceAccount.create=false"

# update ECR_HOST as needed
ECR_HOST=0000000000.dkr-ecr-fips.us-west-2.on.aws

# should use the version of the operator's helm chart, not of the operator itself (eg 1.20930.1, not main-1.20930.1)
OPERATOR_VERSION=1.20930.1

# image tag: the FIPS operator build (note the main- prefix and the -fips suffix).
# Always set this explicitly: the FIPS variant is selected by the tag, so an
# unset tag will not give you a FIPS build.
OPERATOR_IMAGE_TAG=main-1.20930.1-fips

# set AZs as determined by VPC subnets
AZ_LIST='["us-west-2a","us-west-2b","us-west-2c"]'

helm install clickhouse-operator \
   oci://$ECR_HOST/helm/clickhouse-operator-helm \
   --version=$OPERATOR_VERSION \
   --create-namespace \
   -n clickhouse-operator-system \
   --set-json="image.repository=\"$ECR_HOST/clickhouse-operator\"" \
   --set-json="image.tag=\"$OPERATOR_IMAGE_TAG\"" \
   --set-json='cilium.enabled=false' \
   --set-json='idleScalerEnabled=false' \
   --set-json='webhooks.enabled=false' \
   --set-json='operator.debug=true' \
   --set-json='operator.metricsScraper.enabled=false' \
   --set-json="operator.availabilityZones=$AZ_LIST" \
   --set-json='operator.featureFlags.backupOnASeparatePod=true' \
   --set-json='operator.featureFlags.serverCMEKEnabled=true'

CERT_DIR=.
CLUSTER_NAME=default-xx-01

# server
kubectl create secret generic -n ns-${CLUSTER_NAME} ${CLUSTER_NAME}-server-cert-secret \
  --from-file=ca.crt="$CERT_DIR/ca/ca.crt" \
  --from-file=server.crt="$CERT_DIR/server/server.crt" \
  --from-file=server.key="$CERT_DIR/server/server.key" \
  --from-file=client.crt="$CERT_DIR/client/client.crt" \
  --from-file=client.key="$CERT_DIR/client/client.key"

# keeper
kubectl create secret generic -n ns-${CLUSTER_NAME} ${CLUSTER_NAME}-keeper-cert-secret \
  --from-file=ca.crt="$CERT_DIR/ca/ca.crt" \
  --from-file=keeper.crt="$CERT_DIR/keeper/keeper.crt" \
  --from-file=keeper.key="$CERT_DIR/keeper/keeper.key"

# this will be the `default` user's password
PASSWORD='My super secret p@$$w0rd'
if command -v sha256sum &> /dev/null; then
  HASHED_PASSWORD=$(echo -n "$PASSWORD" | sha256sum | awk '{printf $1}' | base64 | tr -d '\n')
else
  HASHED_PASSWORD=$(echo -n "$PASSWORD" | shasum -a 256 | awk '{printf $1}' | base64 | tr -d '\n')
fi

# update values below as needed,
# the ecr host should be the one in your account
# the s3 bucket should be the one created earlier
CLUSTER_NAME=default-xx-01
ECR_HOST=0000000000.dkr-ecr-fips.us-west-2.on.aws
SA_IAM_ROLE_ARN=arn:aws:iam::0000000000:role/CH-S3-${CLUSTER_NAME}-uw2-00-Role

S3_BUCKET_REGION=us-gov-west-1
S3_BUCKET_URL=https://my-clickhouse-data.us-gov-west-1.amazonaws.com
S3_BUCKET_NAME=my-clickhouse-data

# s3 key prefix can use any UUID value, but must be unique for all clusters storing data in the bucket specified above
S3_KEY_PREFIX=ch-s3-$(uuidgen | tr '[:upper:]' '[:lower:]')

# these must match the x86_64 clickhouseGroup labels added to the node groups
SERVER_NG_GROUP=server
KEEPER_NG_GROUP=keeper

# these values should change depending on selected instance sizes
# be sure to take daemon set requirements into account when setting CPU and MEMORY values
SERVER_CPU=64
SERVER_MEMORY=256Gi
# Keeper: high-load profile; for moderate load 2-4 CPU / 8Gi also works (see the infrastructure reference)
KEEPER_CPU=4
KEEPER_MEMORY=16Gi

# Update the `verificationMode` field to control how TLS certificates are verified.
# Defaults to `none`. See Context.h in the ClickHouse source code for more details.
SERVER_VERIFICATION_MODE=none
KEEPER_VERIFICATION_MODE=none

# Currently, usage of `strict` is not recommended and you might encounter issues with the Operator.
# If you require mTLS cert auth, use `relaxed` (Teleport integration requires it)
# SERVER_VERIFICATION_MODE=relaxed
# KEEPER_VERIFICATION_MODE=relaxed

CHART_VERSION=1.8.7

# As a general rule, set cache size to 70-80% of the allocatable instance-store SSD capacity,
# accounting for DaemonSets and cloud-provider reserved disk space.
# The example below sizes the cache for the four 600 GB NVMe devices on m5d.16xlarge.
CACHE_DISK_SIZE_GI=1700Gi

# Select according to the Server node AMI:
# - Bottlerocket with instance-store NVMe bound to /var/lib/kubelet: true
# - AL2023 with instance-store NVMe mounted at /nvme/disk: false
SSD_CACHE_IS_ON_EMPTY_DIR=true

helm install $CLUSTER_NAME \
    oci://$ECR_HOST/helm/onprem-clickhouse-cluster \
    --version=$CHART_VERSION \
    -n ns-$CLUSTER_NAME \
    --create-namespace \
    --set-json="account.hashedPassword=\"$HASHED_PASSWORD\"" \
    --set-json="server.image.repository=\"$ECR_HOST/clickhouse-server\"" \
    --set-json="server.storage.s3.endpoint=\"$S3_BUCKET_URL\"" \
    --set-json="server.storage.s3.region=\"$S3_BUCKET_REGION\"" \
    --set-json="server.storage.s3.bucketName=\"$S3_BUCKET_NAME\"" \
    --set-json="server.storage.s3.keyPrefix=\"$S3_KEY_PREFIX\"" \
    --set-json='server.arm64=false' \
    --set-json="server.image.tag=\"26.2.1.525-fips\"" \
    --set-json="server.podPolicy.nodeSelector.clickhouseGroup=\"$SERVER_NG_GROUP\"" \
    --set-json="server.openSSL.enabled=true" \
    --set-json="server.openSSL.required=true" \
    --set-json="server.openSSL.verificationMode=$SERVER_VERIFICATION_MODE" \
    --set-json="keeper.image.repository=\"$ECR_HOST/clickhouse-keeper\"" \
    --set-json='keeper.arm64=false' \
    --set-json="keeper.image.tag=\"26.2.1.258-fips\"" \
    --set-json="keeper.podPolicy.nodeSelector.clickhouseGroup=\"$KEEPER_NG_GROUP\"" \
    --set-json="keeper.openSSL.enabled=true" \
    --set-json="keeper.openSSL.required=true" \
    --set-json="keeper.openSSL.verificationMode=$KEEPER_VERIFICATION_MODE" \
    --set-json="serviceAccount.annotations={\"eks.amazonaws.com/role-arn\":\"$SA_IAM_ROLE_ARN\"}" \
    --set-json='server.tolerations=[{"effect":"NoSchedule","key":"clickhouse.com/do-not-schedule","operator":"Exists"}]' \
    --set-json='keeper.tolerations=[{"effect":"NoSchedule","key":"clickhouse.com/do-not-schedule","operator":"Exists"}]' \
    --set-json="server.ssdCacheConfiguration.cacheDiskSize=\"$CACHE_DISK_SIZE_GI\"" \
    --set-json="server.ssdCacheConfiguration.isOnEmptyDir=$SSD_CACHE_IS_ON_EMPTY_DIR" \
    --set-json="server.podPolicy.resources.limits.cpu=\"$SERVER_CPU\"" \
    --set-json="server.podPolicy.resources.limits.memory=\"$SERVER_MEMORY\"" \
    --set-json="server.podPolicy.resources.requests.cpu=\"$SERVER_CPU\"" \
    --set-json="server.podPolicy.resources.requests.memory=\"$SERVER_MEMORY\"" \
    --set-json="keeper.podPolicy.resources.limits.cpu=\"$KEEPER_CPU\"" \
    --set-json="keeper.podPolicy.resources.limits.memory=\"$KEEPER_MEMORY\"" \
    --set-json="keeper.podPolicy.resources.requests.cpu=\"$KEEPER_CPU\"" \
    --set-json="keeper.podPolicy.resources.requests.memory=\"$KEEPER_MEMORY\""

kubectl port-forward svc/c-default-xx-01-server-any 9440:9440 -n ns-default-xx-01

<config>
    <openSSL>
        <client>
            <caConfig>/path/to/certs/ca.crt</caConfig>
        </client>
    </openSSL>
</config>

clickhouse client --host localhost --port 9440 --password $PASSWORD --secure --config=client-config.xml

clickhouse-cloud :) select 1;

SELECT 1
Query id: 825591bf-a8e5-4995-ac9f-afb864854ba2
   ┌─1─┐
1. │ 1  │
   └───┘
1 row in set. Elapsed: 0.001 sec.