Skip to main content

Karpenter


Prerequisites

진행 전 Spot Role에 대한 설정을 먼저 해주세요.

Karpenter Role

name = "eks-karpenter-controller-policy"
karpenter_controller_policy = aws.iam.Policy(
name,
name_prefix=name,
policy=json.dumps(
{
"Statement": [
{
"Action": [
"ec2:CreateLaunchTemplate",
"ec2:CreateFleet",
"ec2:RunInstances",
"ec2:CreateTags",
"iam:PassRole",
"ec2:TerminateInstances",
"ec2:DescribeLaunchTemplates",
"ec2:DeleteLaunchTemplate",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeImages",
"ec2:DescribeInstances",
"ec2:DescribeInstanceTypes",
"ec2:DescribeInstanceTypeOfferings",
"ec2:DescribeAvailabilityZones",
"ssm:GetParameter",
"pricing:GetProducts",
],
"Effect": "Allow",
"Resource": "*",
}
],
"Version": "2012-10-17",
}
),
opts=pulumi.ResourceOptions(parent=eks.cluster),
tags={
"Name": name,
"Stack": variable.stack_name,
"Github": variable.github,
},
)

name = "eks-karpenter-controller-role"
karpenter_controller_role = aws.iam.Role(
name,
name_prefix=name,
assume_role_policy=pulumi.Output.all(oidc_url=eks.oidc_provider.url,).apply(
lambda args: json.dumps(
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Federated": f"arn:aws:iam::{variable.account_id}:oidc-provider/{args['oidc_url']}"
},
"Action": "sts:AssumeRoleWithWebIdentity",
"Condition": {
"StringEquals": {
f"{args['oidc_url']}:sub": "system:serviceaccount:karpenter:karpenter"
}
},
}
],
}
)
),
opts=pulumi.ResourceOptions(parent=eks.cluster),
tags={
"Name": name,
"Stack": variable.stack_name,
"Github": variable.github,
},
)

aws.iam.RolePolicyAttachment(
"eks-karpenter-rpa-0",
policy_arn=karpenter_controller_policy.arn,
role=karpenter_controller_role.name,
opts=pulumi.ResourceOptions(parent=karpenter_controller_role),
)

NodeGroup Role

name = "eks-karpenter-ng-role"
ng_role = aws.iam.Role(
name,
name_prefix=name,
assume_role_policy=json.dumps(
{
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {"Service": "ec2.amazonaws.com"},
}
],
"Version": "2012-10-17",
}
),
opts=pulumi.ResourceOptions(parent=eks.cluster),
tags={
"Name": name,
"Stack": variable.stack_name,
"Github": variable.github,
},
)

node_policy_arns = {
"0": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
"1": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
"2": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
"3": "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", # Karpenter
}

ng_rpas = []
for i, arn in node_policy_arns.items():
ng_rpas.append(
aws.iam.RolePolicyAttachment(
f"eks-karpenter-ng-rpa-{i}",
policy_arn=arn,
role=ng_role.name,
opts=pulumi.ResourceOptions(parent=ng_role),
)
)

name = "eks-karpenter-ng-profile"
default_ng_profile = aws.iam.InstanceProfile(
name,
name_prefix=name,
role=ng_role.name,
opts=pulumi.ResourceOptions(parent=ng_role),
tags={
"Name": name,
"Stack": variable.stack_name,
"Github": variable.github,
},
)

Installation

danger

설치 전에 CoreDNS가 작동하는 지 확인하시기 바랍니다.

helm repo add karpenter https://charts.karpenter.sh
mkdir -p node/karpenter/helm
helm repo update karpenter \
&& helm search repo karpenter/karpenter -l | head -n 10
helm show values karpenter/karpenter \
--version 0.16.1 \
> node/karpenter/helm/values.yaml
serviceAccount:
annotations:
eks.amazonaws.com/role-arn: arn:aws:iam::<account_id>:role/<role_name>

affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: karpenter.sh/provisioner-name
operator: DoesNotExist
- key: eks-ng
operator: Exists

tolerations:
- key: eks-ng
operator: Exists
effect: NoSchedule

controller:
# Pending Pod가 새로 생성되면 batch를 만들려고 시도함
# 새 Pending Pod가 생기고 batchIdleDuration 시간 만큼 대기
# 그 안에 새로운 Pending Pod가 생기면 batch에 포함 시키고 다시 batchIdleDuration 시간 대기
# 이 과정이 반복되다가 마지막 Pending Pod가 생긴 후 batchIdleDuration을 넘기거나
# 첫 Pending Pod가 생긴 후 batchMaxDuration을 넘기면 batch만들기 종료
# https://github.com/aws/karpenter/blob/main/pkg/controllers/provisioning/batcher.go
batchMaxDuration: 10s
batchIdleDuration: 1s

clusterName: <cluster_name>

clusterEndpoint: <cluster_endpoint>

aws:
defaultInstanceProfile: <instance_profile>
kubectl create namespace karpenter
helm upgrade karpenter karpenter/karpenter \
--install \
--version 0.16.1 \
-n karpenter \
-f node/karpenter/helm/values.yaml

Reference