Ansible для управления несколькими k3s-кластерами
Published: 2026-05-01
Ansible берёт на себя всё до Flux: подготовку ОС, установку k3s, bootstrap Cilium, получение kubeconfig. Каждый плейбук идемпотентен — повторный запуск на уже существующем кластере безопасен и служит основным способом применения изменений на уровне ОС.
Структура inventory
ansible/
inventory/
dev/
hosts.yaml
test/
hosts.yaml
group_vars/
all.yaml # общие переменные: версии k3s, Cilium
dev.yaml # переопределения для dev
test.yaml
roles/
node-prepare/
k3s-master/
k3s-worker/
get-kubeconfig/
bootstrap-cilium/
flux-bootstrap/
install-k3s.yaml
upgrade-k3s.yaml
get-kubeconfig.yaml
upgrade-cilium.yaml
bootstrap-flux.yaml
hosts.yaml
yamlall:
children:
masters:
hosts:
dev-master-01:
ansible_host: 10.0.1.10
ansible_user: ubuntu
ansible_ssh_private_key_file: ~/.ssh/infra_ed25519
workers:
hosts:
dev-worker-01:
ansible_host: 10.0.1.11
ansible_user: ubuntu
ansible_ssh_private_key_file: ~/.ssh/infra_ed25519
dev-worker-02:
ansible_host: 10.0.1.12
ansible_user: ubuntu
ansible_ssh_private_key_file: ~/.ssh/infra_ed25519
group_vars/all.yaml
yamlk3s_version: "v1.29.3+k3s1"
cilium_version: "1.15.3"
cilium_cli_version: "v0.16.3"
k3s_server_args:
- "--disable traefik"
- "--disable servicelb"
- "--flannel-backend=none"
- "--disable-network-policy"
- "--cluster-cidr=10.244.0.0/16"
- "--service-cidr=10.96.0.0/12"
Traefik и servicelb отключены, потому что Cilium берёт на себя ingress (через APISIX) и балансировку нагрузки (через L2 announcements). Flannel тоже отключён — Cilium заменяет его как CNI.
install-k3s.yaml
yaml- name: Prepare nodes
hosts: all
become: true
roles:
- role: node-prepare # swap off, sysctl, kernel modules
- name: Install k3s master
hosts: masters
become: true
roles:
- role: k3s-master
- name: Install k3s workers
hosts: workers
become: true
roles:
- role: k3s-worker
- name: Bootstrap Cilium
hosts: masters
become: false
roles:
- role: bootstrap-cilium
Роль node-prepare
Загружает нужные модули ядра и задаёт sysctl-параметры:
yaml# roles/node-prepare/tasks/main.yaml
- name: Disable swap
command: swapoff -a
when: ansible_swaptotal_mb > 0
- name: Remove swap from fstab
lineinfile:
path: /etc/fstab
regexp: '.*swap.*'
state: absent
- name: Load required kernel modules
modprobe:
name: "{{ item }}"
state: present
loop:
- overlay
- br_netfilter
- name: Persist kernel modules
copy:
dest: /etc/modules-load.d/k8s.conf
content: |
overlay
br_netfilter
- name: Set sysctl values
sysctl:
name: "{{ item.key }}"
value: "{{ item.value }}"
sysctl_set: true
reload: true
loop:
- {key: "net.bridge.bridge-nf-call-iptables", value: "1"}
- {key: "net.bridge.bridge-nf-call-ip6tables", value: "1"}
- {key: "net.ipv4.ip_forward", value: "1"}
Роль k3s-master
yaml# roles/k3s-master/tasks/main.yaml
- name: Check if k3s is already installed
stat:
path: /usr/local/bin/k3s
register: k3s_binary
- name: Install k3s master
shell: |
curl -sfL https://get.k3s.io | \
INSTALL_K3S_VERSION="{{ k3s_version }}" \
K3S_TOKEN="{{ k3s_token }}" \
sh -s - server {{ k3s_server_args | join(' ') }}
when: not k3s_binary.stat.exists
- name: Wait for k3s to be ready
command: k3s kubectl get nodes
register: nodes_ready
until: nodes_ready.rc == 0
retries: 20
delay: 10
Роль bootstrap-cilium
Cilium нужно установить до планирования любых подов (CoreDNS не стартует без CNI):
yaml# roles/bootstrap-cilium/tasks/main.yaml
- name: Install cilium-cli
get_url:
url: "https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-amd64.tar.gz"
dest: /tmp/cilium-cli.tar.gz
- name: Extract cilium-cli
unarchive:
src: /tmp/cilium-cli.tar.gz
dest: /usr/local/bin
remote_src: true
- name: Check if Cilium is already installed
command: cilium status
environment:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
register: cilium_status
failed_when: false
- name: Install Cilium
command: >
cilium install
--version {{ cilium_version }}
--set kubeProxyReplacement=true
--set ipam.mode=kubernetes
--set k8sServiceHost=127.0.0.1
--set k8sServicePort=6443
environment:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
when: "'cilium' not in cilium_status.stdout"
- name: Wait for Cilium to be ready
command: cilium status --wait
environment:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
retries: 5
delay: 30
get-kubeconfig.yaml
yaml- name: Fetch kubeconfig from master
hosts: masters
tasks:
- name: Slurp kubeconfig
slurp:
src: /etc/rancher/k3s/k3s.yaml
register: kubeconfig_b64
- name: Save locally
delegate_to: localhost
copy:
content: "{{ kubeconfig_b64.content | b64decode | replace('127.0.0.1', ansible_host) }}"
dest: "{{ playbook_dir }}/../.kubeconfigs/{{ inventory_hostname | split('-') | first }}.yaml"
replace('127.0.0.1', ansible_host) заменяет loopback-адрес реальным IP ноды, чтобы kubeconfig работал за пределами кластера.
upgrade-k3s.yaml
yaml- name: Upgrade k3s
hosts: masters
become: true
serial: 1 # обновлять по одной ноде
tasks:
- name: Drain node
command: k3s kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-emptydir-data
delegate_to: "{{ groups['masters'][0] }}"
- name: Upgrade k3s binary
shell: |
curl -sfL https://get.k3s.io | \
INSTALL_K3S_VERSION="{{ k3s_version }}" \
sh -s - server {{ k3s_server_args | join(' ') }}
- name: Uncordon node
command: k3s kubectl uncordon {{ inventory_hostname }}
delegate_to: "{{ groups['masters'][0] }}"
Запуск
bash# Первоначальная установка кластера
ansible-playbook install-k3s.yaml -i inventory/dev/
# Получить kubeconfig для запечатывания
ansible-playbook get-kubeconfig.yaml -i inventory/dev/
# Обновить версию k3s (предварительно обновить k3s_version в group_vars)
ansible-playbook upgrade-k3s.yaml -i inventory/dev/
# Bootstrap Flux на новом кластере
ansible-playbook bootstrap-flux.yaml -i inventory/dev/
Запуск только для конкретного хоста:
bashansible-playbook install-k3s.yaml -i inventory/dev/ --limit dev-worker-02