0%
February 13, 2025

Self-Managed Kubenetes Cluster Using k3s, EC2 and RDS

terraform

Contents

Why Still EC2?

Nowadays we don't manage EC2s on our own due to the advent of ECS and the flexible scaling policies. However, an EC2 may still be useful in certain scenarios such as acting as a bastion host for an RDS instance sitting inside of a private VPC.

Let's study how to define EC2 and let's define a cluster of EC2s as a module, of which every EC2 instance executes a k3s associated with a common RDS instance to share kubenetes state, these EC2s will form a small worker node.

Entrypoint --- main.tf

module "networking" {
  source                 = "./modules/networking"
  vpc_cidr               = "10.123.0.0/16"                                     // this is used to create a custom VPC
  public_cidrs           = ["10.123.2.0/24", "10.123.4.0/24"]                  // this is for public subnets
  private_cidrs          = ["10.123.1.0/24", "10.123.3.0/24", "10.123.5.0/24"] // this is for privae subnets
  ssh_access_ip          = var.ssh_access_ip
  create_db_subnet_group = true
}
module "database" {
  source                 = "./modules/database"
  db_storage             = 10
  db_engine              = "mysql"
  db_engine_version      = "5.7.44"
  db_instance_class      = "db.t3.micro"
  db_name                = var.db_name
  db_username            = var.db_username
  db_password            = var.db_password
  db_identifier          = "james-love-love-db"
  skip_final_snapshot    = true
  db_subnet_group_name   = length(module.networking.db_subnet_group_names) == 1 ? module.networking.db_subnet_group_names[0] : ""
  vpc_security_group_ids = [module.networking.db_security_group_id]
}

module "loadbalancing" {
  source                 = "./modules/loadbalancing"
  public_security_groups = [module.networking.public_http_sg.id, module.networking.public_ssh_sg.id]
  public_subnets         = module.networking.public_subnet_ids
  tg_port                = 8000
  tg_protocol            = "HTTP"
  vpc_id                 = module.networking.vpc_id
  lb_healthy_threshold   = 2
  lb_unhealthy_threshold = 2
  lb_timeout             = 3
  lb_interval            = 30
  listener_port          = 8000
  listener_portocol      = "HTTP"
}

module "compute" {
  source                = "./modules/compute"
  instance_count        = 1
  instance_type         = "t3.micro"
  vol_size              = 10
  ec2_security_group_id = module.networking.james_ec2_sg.id
  public_security_gp_ids = [
    module.networking.public_ssh_sg.id,
    module.networking.public_http_sg.id
  ]
  public_subnet_ids = module.networking.public_subnet_ids
  key_name          = "james_ec2_key"
  # public_key_path       = "C:\\Users\\machingclee\\.ssh\\jameskey.pub" # windows specific
  # generated by ssh-keygen -t ed25519 -f ~/.ssh/james_ec2_key
  public_key_path        = "/Users/chingcheonglee/.ssh/james_ec2_key.pub"
  db_endpoint            = module.database.db_endpoint
  db_name                = var.db_name
  db_password            = var.db_password
  db_user                = var.db_username
  user_data_path         = "${path.root}/userdata.tpl"
  james_target_group_arn = module.loadbalancing.james_target_group_arn
}

Compute Module

compute/main.tf
data "aws_ami" "server_ami" {
  most_recent = true
  # it can be found in ec2 > Images > AMIs > Public Images
  owners = ["099720109477"]
  filter {
    name   = "name"
    values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"]
  }
}

resource "random_id" "james_node_id" {
  byte_length = 2
  count       = var.instance_count
  # this acts like an identifier in a map, and will create another
  # random_id with the same key_name

  /*
    The random_id resource with count will generate different random IDs, but it won't create additional instances by itself. The number of instances is controlled by var.instance_count.
    If var.instance_count = 1, you'll only get one instance regardless of random IDs. To create more instances:
    */

  keepers = {
    key_name = var.key_name
  }
}

resource "aws_key_pair" "james_auth" {
  key_name   = var.key_name
  public_key = file(var.public_key_path)
}

resource "aws_instance" "james_node" {
  count                  = var.instance_count # 1
  instance_type          = var.instance_type  # t3.micro
  security_groups        = [var.ec2_security_group_id]
  ami                    = data.aws_ami.server_ami.id
  key_name               = aws_key_pair.james_auth.key_name
  vpc_security_group_ids = var.public_security_gp_ids
  subnet_id              = var.public_subnet_ids[count.index]
  user_data = templatefile(var.user_data_path, {
    nodename    = "james-node-${random_id.james_node_id[count.index].dec}"
    dbuser      = var.db_user
    dbpass      = var.db_password
    db_endpoint = var.db_endpoint
    dbname      = var.db_name
  })
  root_block_device {
    volume_size = var.vol_size # 10
  }
  tags = {
    Name = "james_node-${random_id.james_node_id[count.index].dec}"
  }
}

# attach EC2 instances (target_id's) to the target group (target_group_arn)
resource "aws_lb_target_group_attachment" "james_tg_attach" {
  count            = var.instance_count
  target_group_arn = var.james_target_group_arn
  target_id        = aws_instance.james_node[count.index].id
  // port = 8000 target port of the EC2, this will override the port in target group on a per instance basis
}
compute/outputs.tf
output "instance" {
  value     = aws_instance.james_node[*]
  sensitive = true
}
compute/variables.tf
variable "instance_count" {
  type = number
}

variable "instance_type" {
  type = string
}

variable "public_security_gp_ids" {
  type = list(string)
}

variable "public_subnet_ids" {
  type = list(string)
}

variable "vol_size" {
  type = number
}

variable "key_name" {
  type = string
}

variable "public_key_path" {
  type = string
}

variable "user_data_path" {
  type = string
}

variable "db_user" {
  type = string
}
variable "db_password" {
  type = string
}
variable "db_endpoint" {
  type = string
}
variable "db_name" {
  type = string
}

variable "ec2_security_group_id" {
  type = string
}

variable "james_target_group_arn" {
  type = string
}

Database Module

database/main.tf
resource "aws_db_instance" "james_db" {
  allocated_storage      = var.db_storage
  engine                 = var.db_engine
  engine_version         = var.db_engine_version
  instance_class         = var.db_instance_class
  db_name                = var.db_name
  username               = var.db_username
  password               = var.db_password
  db_subnet_group_name   = var.db_subnet_group_name
  vpc_security_group_ids = var.vpc_security_group_ids // the RDS instance must be assigned at least one security group.
  identifier             = var.db_identifier
  skip_final_snapshot    = var.skip_final_snapshot
  tags = {
    Name = "james-db"
  }
}
database/outputs.tf
output "db_endpoint" {
  value = aws_db_instance.james_db.endpoint
}
database/variables.tf
variable "db_storage" {
  type        = number
  description = "Should be an integer in GB"
}

variable "db_engine" {
  type    = string
  default = "mysql"
}

variable "db_engine_version" {
  type = string
}

variable "db_instance_class" {
  type = string
}

variable "db_name" {
  type = string
}

variable "db_username" {
  type = string
}

variable "db_password" {
  type = string
}

variable "db_subnet_group_name" {
  type = string
}

variable "vpc_security_group_ids" {
  type = list(string)
}

variable "db_identifier" {
  type = string
}

variable "skip_final_snapshot" {
  type = bool
}

Load Balancer Module

loadbalancing/main.tf

Here the binding (attachment) of aws_lb_target_group.james_target_group is done in compute/main.tf:

resource "aws_lb" "james_lb" {
  name            = "james-loadbalancer"
  subnets         = var.public_subnets
  security_groups = var.public_security_groups
  idle_timeout    = 900
}

/*
Avoid naming conflicts during recreate/redeploy:
When you destroy and recreate resources, AWS keeps the old name in a "cooling period"
Without random suffixes, you might get errors like "name already exists" when redeploying
*/

resource "aws_lb_target_group" "james_target_group" {
  name     = "james-lb-tg-${substr(uuid(), 0, 4)}"
  port     = var.tg_port
  protocol = var.tg_protocol
  vpc_id   = var.vpc_id
  lifecycle {
    ignore_changes        = [name]
    create_before_destroy = true
    // when set to false, the destruction will be halted because:
    // when we change the port, the target group will be destroyed,
    // and the listener has no where to route the traffic until the new target group is created
    // but listener cannot live without target group, the deletion of the target group will get halted due to AWS's own validation
    // we use create_before_destroy = true to make sure there must be an existing target group assignable to the listener
  }
  health_check {
    healthy_threshold   = var.lb_healthy_threshold
    unhealthy_threshold = var.lb_unhealthy_threshold
    timeout             = var.lb_timeout
    interval            = var.lb_interval
  }
}

resource "aws_lb_listener" "james_lb_listener" {
  load_balancer_arn = aws_lb.james_lb.arn
  port              = var.listener_port
  protocol          = var.listener_portocol
  default_action {
    type             = "forward"
    target_group_arn = aws_lb_target_group.james_target_group.arn
  }
}
loadbalancing/outputs.tf
output "james_target_group_arn" {
  value = aws_lb_target_group.james_target_group.arn
}

output "lb_endpoint" {
  value = aws_lb.james_lb.dns_name
}
loadbalancing/variables.tf
variable "public_security_groups" {
  type = list(string)
}

variable "public_subnets" {
  type = list(string)
}

variable "tg_port" {
  type = string
}
variable "tg_protocol" {
  type = string
}

variable "vpc_id" {
  type = string
}

variable "lb_healthy_threshold" {
  type = string
}

variable "lb_unhealthy_threshold" {
  type = string
}

variable "lb_timeout" {
  type = number
}

variable "lb_interval" {
  type = number
}

variable "listener_port" {
  type = number
}

variable "listener_portocol" {
  type = string
}

Network Module

network/main.tf
resource "random_integer" "random" {
  min = 1
  max = 100
}

data "aws_availability_zones" "available" {
  state = "available"
}

resource "aws_vpc" "james_vpc" {
  cidr_block           = var.vpc_cidr
  enable_dns_hostnames = true
  enable_dns_support   = true

  tags = {
    Name = "james_vpc-${random_integer.random.id}"
  }

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_route_table_association" "james_public_subnet_association" {
  count          = length(var.public_cidrs)
  subnet_id      = aws_subnet.james_public_subnet.*.id[count.index]
  route_table_id = aws_route_table.james_public_route_table.id
}


resource "aws_subnet" "james_public_subnet" {
  count                   = length(var.public_cidrs)
  vpc_id                  = aws_vpc.james_vpc.id
  cidr_block              = var.public_cidrs[count.index]
  map_public_ip_on_launch = true
  availability_zone       = data.aws_availability_zones.available.names[count.index]

  tags = {
    Name = "james_public_subnet_${count.index + 1}"
  }
}


resource "aws_subnet" "james_private_subnet" {
  count                   = length(var.private_cidrs)
  vpc_id                  = aws_vpc.james_vpc.id
  cidr_block              = var.private_cidrs[count.index]
  map_public_ip_on_launch = false
  availability_zone       = data.aws_availability_zones.available.names[count.index]

  tags = {
    Name = "james_private_subnet_${count.index + 1}"
  }
}

resource "aws_internet_gateway" "james_internet_gateway" {
  vpc_id = aws_vpc.james_vpc.id
  tags = {
    Name = "james_igw"
  }
}

resource "aws_route_table" "james_public_route_table" {
  vpc_id = aws_vpc.james_vpc.id
  tags = {
    Name = "james_public_route_table"
  }
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.james_internet_gateway.id
  }
}

# default VPC comes with an Internet Gateway and internet access pre-configured because
# it's designed for immediate use and backward compatibility with EC2-Classic.
# It's meant to help users get started quickly, while custom VPCs follow stricter security practices.

# for new VPC there is no internet gateway and therefore its default route table is a suitable candidate to be assigned
# to private subnet, and we create additional one for public subnet with a record from igw to 0.0.0.0/0
resource "aws_default_route_table" "james_private_route_table" {
  default_route_table_id = aws_vpc.james_vpc.default_route_table_id

  tags = {
    Name = "james_private_route_table"
  }
}

resource "aws_security_group" "james_ssh_sg" {
  name        = "ssh_sg"
  description = "Security Group for SSH Access"
  vpc_id      = aws_vpc.james_vpc.id

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = [var.ssh_access_ip]
    description = "for SSH access"
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_security_group" "james_http_sg" {
  name        = "james_http_sg"
  description = "Security Group for Http"
  vpc_id      = aws_vpc.james_vpc.id

  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "port 80 for public access"
  }

  ingress {
    from_port   = 8000
    to_port     = 8000
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "port 8000 for public access"
  }
}

resource "aws_security_group" "ec2_security_group" {
  name        = "james_ec2_sg"
  description = "Security Group for EC2"
  vpc_id      = aws_vpc.james_vpc.id
}


resource "aws_security_group" "james_private_rds" {
  name        = "james_private_rds"
  description = "Security Group for Private RDS"
  vpc_id      = aws_vpc.james_vpc.id

  ingress {
    from_port   = 3306
    to_port     = 3306
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "pgsql allows everything in the VPC to access"
  }

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_db_subnet_group" "james_rds_subnet_group" {
  count      = var.create_db_subnet_group == true ? 1 : 0
  name       = "james_rds_subnet_group"
  subnet_ids = aws_subnet.james_private_subnet.*.id
  tags = {
    Name = "james_rds_subnet_group"
  }
}
network/outputs.tf
output "vpc_id" {
  value = aws_vpc.james_vpc.id
}

output "db_subnet_group_names" {
  value = aws_db_subnet_group.james_rds_subnet_group.*.name
}

output "db_security_group_id" {
  value = aws_security_group.james_private_rds.id
}

output "public_subnet_ids" {
  value = aws_subnet.james_public_subnet.*.id
}

output "public_http_sg" {
  value = aws_security_group.james_http_sg
}

output "public_ssh_sg" {
  value = aws_security_group.james_ssh_sg
}

output "james_ec2_sg" {
  value = aws_security_group.ec2_security_group
}
network/variables.tf
variable "vpc_cidr" {
  type = string
}

variable "public_cidrs" {
  type = list(string)
}


variable "private_cidrs" {
  type = list(string)
}

variable "ssh_access_ip" {
  type = string
}

variable "create_db_subnet_group" {
  type = bool
}

<project-root>/userdata.tpl

#!/bin/bash
sudo hostnamectl set-hostname ${nodename} &&
curl -sfL https://get.k3s.io | sh -s - server \
--datastore-endpoint="mysql://${dbuser}:${dbpass}@tcp(${db_endpoint})/${dbname}" \
--write-kubeconfig-mode 644 \
--tls-san=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) \
--token="th1s1sat0k3n!"

References