I wanted to run Microk8s on a Proxmox 6 host inside of an LXC container. These are my notes from the journey.
- Create a privileged LXC container through the Proxmox web interface
- Enable nesting and FUSE
- In Proxmox UI, select container, then Options > Features > Check nesting and FUSE boxes
- SSH into the Proxmox host and edit the container's config in /etc/pve/lxc/.conf
- Add the following lines
- lxc.apparmor.profile: unconfined
- lxc.cap.drop:
- lxc.mount.auto: proc:rw sys:rw
- Add the following lines
- Start (or restart) the container
- SSH into the container and create a symlink for /dev/kmsg, which is missing in Ubuntu 19.10 containers
- ln -s /dev/console /dev/kmsg
- Has to be repeated on container reboot, which is annoying.
- Install snapd: apt install snapd
- Install microk8s: snap install microk8s --classic
The snap commands may need to be run more than once to get past errors.
If you get "cannot change profile for the next exec call: No such file or directory", try running: apparmor_parser -r /var/lib/snapd/apparmor/profiles/*
Here's what works for me:
`─➤ cat ansible/play_deploy_k8s_create-lxc.yml
name: Setup LXC-Container (K8S) on PVE for deploying of K8S on it
hosts: one-bn-pve1
gather_facts: yes
vars:
ansible_python_interpreter: "/usr/bin/python3"
username: "root@pam"
user_password: ""
lxc_root_password: "FOO"
api_host: "10.10.xxx.yy"
netif: '{"net0":"name=eth0,ip=dhcp,ip6=dhcp,bridge=vmbr0"}'
os_template_name: "debian-12-standard_12.2-1_amd64.tar.zst"
vmid: 180
node: pve1
hostname: "k8s-{{ vmid }}"
cpu_cores: 2
memory: 1024
rootfs_disk_size: "8"
storage: "Ceph_1TB"
module_defaults:
ansible.builtin.get_url:
validate_certs: false
tasks:
name: Install Proxmoxer"
ansible.builtin.apt:
name: python3-proxmoxer
state: latest
update_cache: true
name: Check if LXC template is downloaded
ansible.builtin.stat:
path: "/var/lib/vz/template/cache/{{ os_template_name }}"
register: p
ansible.builtin.debug: msg="Template was already downloaded. Skipping."
when: p.stat.exists
name: Show IP
ansible.builtin.shell: ip a
name: Download LXC template
ansible.builtin.shell: pveam download local {{ os_template_name }}
when: p.stat.exists == False
name: Create K8S LXC-Container
connection: local
community.general.proxmox:
vmid: "{{ vmid }}"
node: "{{ node }}"
state: "present"
purge: true
onboot: true
api_user: "{{ username }}"
api_password: "{{ user_password }}"
api_host: "{{ api_host }}"
password: "{{ lxc_root_password }}"
hostname: "{{ hostname }}-pve1"
ostemplate: "local:vztmpl/{{ os_template_name }}"
storage: "{{ storage }}"
netif: "{{ netif }}"
memory: "{{ memory }}"
disk: "{{ rootfs_disk_size }}"
swap: "0"
unprivileged: false
features:
pubkey: ""
register: container_present
tags:
name: Check if K8S LXC is running
ansible.builtin.shell: pct status {{ vmid }}
register: lxc_running
tags:
name: Insert lines into LXC conf
ansible.builtin.blockinfile:
path: "/etc/pve/lxc/{{ vmid }}.conf"
block: |
lxc.apparmor.profile: unconfined
lxc.cap.drop:
lxc.mount.auto: proc:rw sys:rw
lxc.mount.entry: /sys/kernel/security sys/kernel/security none bind,create=file 0 0
backup: yes
when: '"status: running" not in lxc_running.stdout'
tags:
name: Start K8S LXC container
community.general.proxmox:
vmid: "{{ vmid }}"
node: "{{ node }}"
state: "started"
api_user: "{{ username }}"
api_password: "{{ user_password }}"
api_host: "{{ api_host }}"
when: '"status: running" not in lxc_running.stdout'
tags:
name: get K8S LXC Container IP
ansible.builtin.shell:
cmd: pct exec {{ vmid }} ip addr show|grep 'inet 10'| awk -F ' ' '{print $2}'
register: lxcip
name: show K8S LXC Container IP
ansible.builtin.debug:
msg: The Container {{ vmid }} has the following IP {{ lxcip.stdout }}
when: lxcip.stdout != "" %`
`- hosts: "{{ machines }}"
#remote_user: "{{ remote_user }}"
tasks:
name: Add CronJob "@reboot ln -s /dev/console /dev/kmsg"
ansible.builtin.cron:
name: "create /dev/kmsg"
special_time: reboot
job: "ln -s /dev/console /dev/kmsg"
name: Install required Packages
apt:
pkg:
state: latest
update_cache: true
name: Reboot
ansible.builtin.reboot:
msg: "Reboot initiated by Ansible"
connect_timeout: 5
reboot_timeout: 600
pre_reboot_delay: 0
post_reboot_delay: 30
test_command: whoami
name: Install microk8s
community.general.snap:
name: microk8s
state: absent
classic: true`
'- hosts: all
vars:
ansible_ssh_private_key_file: ~/.ssh/key
ansible_ssh_user: root
tasks:
name: Write the LXCs host key to known_hosts
connection: local
shell: "ssh-keyscan -H {{ inventory_hostname }} >> ~/.ssh/known_hosts"
name: Add CronJob "@reboot ln -s /dev/console /dev/kmsg"
ansible.builtin.cron:
name: "create /dev/kmsg"
special_time: reboot
job: "ln -s /dev/console /dev/kmsg"
name: Install required Packages
apt:
pkg:
state: latest
update_cache: true
name: Reboot
ansible.builtin.reboot:
msg: "Reboot initiated by Ansible"
connect_timeout: 5
reboot_timeout: 30
pre_reboot_delay: 0
post_reboot_delay: 10
test_command: whoami
name: Install microk8s
community.general.snap:
name: microk8s
state: present
classic: true'
Run them in the listed order and you'll have your 1st microk8s LXC