neoaggelos · January 19, 2025 22:58 · wirwolf · Jul 9, 2024 · neoaggelos · Jul 11, 2024
diff --git a/cluster.yaml b/cluster.yaml
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
  name: test
  namespace: default
 spec:
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    kind: MicroK8sControlPlane
    name: test-control-plane
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
    kind: LXDCluster
    name: test
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: LXDCluster
 metadata:
  name: test
  namespace: default
 spec: {}
 ---
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 kind: MicroK8sControlPlane
 metadata:
  name: test-control-plane
  namespace: default
 spec:
  controlPlaneConfig:
    clusterConfiguration:
      portCompatibilityRemap: true
    initConfiguration:
      IPinIP: true
      addons:
      - dns
      - ingress
      joinTokenTTLInSecs: 9000
  machineTemplate:
    infrastructureTemplate:
      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
      kind: LXDMachineTemplate
      name: test-control-plane
  replicas: 1
  version: v1.25.0
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: LXDMachineTemplate
 metadata:
  name: test-control-plane
  namespace: default
 spec:
  template:
    spec:
      imageAlias: u22
      instanceType: container
      profiles:
      - default
      - microk8s
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
 metadata:
  name: test-md-0
  namespace: default
 spec:
  clusterName: test
  replicas: 0
  selector:
    matchLabels: null
  template:
    spec:
      bootstrap:
        configRef:
          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
          kind: MicroK8sConfigTemplate
          name: test-md-0
      clusterName: test
      infrastructureRef:
        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
        kind: LXDMachineTemplate
        name: test-md-0
      version: 1.25.0
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: LXDMachineTemplate
 metadata:
  name: test-md-0
  namespace: default
 spec:
  template:
    spec:
      imageAlias: u22
      instanceType: container
      profiles:
      - default
      - microk8s
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: MicroK8sConfigTemplate
 metadata:
  name: test-md-0
  namespace: default
 spec:
  template:
    spec: {}
diff --git a/init b/init
 # configure LXD, and allow access over HTTPS.
 # note the IP address (in this case, 10.0.3.181) and the trust password.
 $ sudo lxd init
 Would you like to use LXD clustering? (yes/no) [default=no]: yes
 What IP address or DNS name should be used to reach this node? [default=10.0.3.181]: 
 Are you joining an existing cluster? (yes/no) [default=no]: 
 What name should be used to identify this node in the cluster? [default=test-ovn]: 
 Setup password authentication on the cluster? (yes/no) [default=no]: yes
 Trust password for new clients: 
 Again: 
 ....

 # create image alias for ubuntu 22.04 
 $ sudo lxc launch ubuntu:22.04 t1
 $ sudo lxc image alias create u22 ubuntu:22.04
 $ sudo lxc rm t1 --force

 # create profile
 $ lxc profile create microk8s
 $ curl https://raw.githubusercontent.com/ubuntu/microk8s/master/tests/lxc/microk8s.profile | lxc profile edit microk8s

 # deploy LXD provider and configure access to server (replace IP and password)
 $ microk8s kubectl apply -f provider.yaml
 $ microk8s kubectl create configmap -n capl-system lxd-socket --from-literal=LXD_SERVER=https://10.0.3.181:8443 --from-literal=LXD_PASSWORD=password

 # deploy cluster.yaml. initially, it has 1 control plane node and 0 workers
 $ microk8s kubectl apply -f cluster.yaml

 # wait for init node to come up, check with 'lxc list' and note its IP address
 $ lxc list
 +-------------------------------+---------+-----------------------------+------+-----------+-----------+
 |             NAME              |  STATE  |            IPV4             | IPV6 |   TYPE    | SNAPSHOTS |
 +-------------------------------+---------+-----------------------------+------+-----------+-----------+
 | test-test-control-plane-pkstp | RUNNING | 10.0.0.187 (eth0)           |      | CONTAINER | 0         |
 +-------------------------------+---------+-----------------------------+------+-----------+-----------+

 # after deployment, fix `test-kubeconfig` secret to unblock the control plane provider
 $ microk8s kubectl edit cluster test                # change control api endpoint to 10.0.0.187:6443
 $ microk8s kubectl edit lxdcluster test             # change control api endpoint to 10.0.0.187:6443
 $ clusterctl get kubeconfig test > kubeconfig
 $ vim kubeconfig                                    # change 'https://TODO:12345' to 'https://10.0.0.187:6443' 
 $ cat kubeconfig | base64 -w0
 $ microk8s kubectl edit secret test-kubeconfig      # change value to the new base64 string

 # cluster is now ready to scale. edit 'cluster.yaml', change 'replicas: 0' and re-apply to deploy worker nodes
diff --git a/provider.yaml b/provider.yaml
 apiVersion: v1
 kind: Namespace
 metadata:
  labels:
    control-plane: controller-manager
  name: capl-system
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.9.0
  creationTimestamp: null
  labels:
    cluster.x-k8s.io/provider: infrastructure-lxd
    cluster.x-k8s.io/v1beta1: v1alpha1
  name: lxdclusters.infrastructure.cluster.x-k8s.io
 spec:
  group: infrastructure.cluster.x-k8s.io
  names:
    categories:
    - cluster-api
    kind: LXDCluster
    listKind: LXDClusterList
    plural: lxdclusters
    shortNames:
    - lc
    singular: lxdcluster
  scope: Namespaced
  versions:
  - additionalPrinterColumns:
    - description: Cluster to which this LXDCluster belongs
      jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name
      name: Cluster
      type: string
    - description: Cluster infrastructure is ready for LXD instances
      jsonPath: .status.ready
      name: Ready
      type: string
    - description: Time duration since creation of LXDCluster
      jsonPath: .metadata.creationTimestamp
      name: Age
      type: date
    name: v1alpha1
    schema:
      openAPIV3Schema:
        description: LXDCluster is the Schema for the lxdclusters API
        properties:
          apiVersion:
            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            description: LXDClusterSpec defines the desired state of LXDCluster
            properties:
              controlPlaneEndpoint:
                description: ControlPlaneEndpoint represents the endpoint to communicate with the control plane.
                properties:
                  host:
                    description: The hostname on which the API server is serving.
                    type: string
                  port:
                    description: The port on which the API server is serving.
                    format: int32
                    type: integer
                required:
                - host
                - port
                type: object
            type: object
          status:
            description: LXDClusterStatus defines the observed state of LXDCluster
            properties:
              ready:
                description: Ready denotes that the LXD cluster (infrastructure) is ready.
                type: boolean
            required:
            - ready
            type: object
        type: object
    served: true
    storage: true
    subresources:
      status: {}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.9.0
  creationTimestamp: null
  labels:
    cluster.x-k8s.io/provider: infrastructure-lxd
    cluster.x-k8s.io/v1beta1: v1alpha1
  name: lxdclustertemplates.infrastructure.cluster.x-k8s.io
 spec:
  group: infrastructure.cluster.x-k8s.io
  names:
    kind: LXDClusterTemplate
    listKind: LXDClusterTemplateList
    plural: lxdclustertemplates
    singular: lxdclustertemplate
  scope: Namespaced
  versions:
  - name: v1alpha1
    schema:
      openAPIV3Schema:
        description: LXDClusterTemplate is the Schema for the lxdclustertemplates API
        properties:
          apiVersion:
            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            description: LXDClusterTemplateSpec defines the desired state of LXDClusterTemplate
            properties:
              template:
                properties:
                  metadata:
                    description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
                    properties:
                      annotations:
                        additionalProperties:
                          type: string
                        description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
                        type: object
                      labels:
                        additionalProperties:
                          type: string
                        description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
                        type: object
                    type: object
                  spec:
                    description: Spec is the specification of the desired behaviour of the cluster.
                    properties:
                      controlPlaneEndpoint:
                        description: ControlPlaneEndpoint represents the endpoint to communicate with the control plane.
                        properties:
                          host:
                            description: The hostname on which the API server is serving.
                            type: string
                          port:
                            description: The port on which the API server is serving.
                            format: int32
                            type: integer
                        required:
                        - host
                        - port
                        type: object
                    type: object
                required:
                - spec
                type: object
            required:
            - template
            type: object
        type: object
    served: true
    storage: true
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.9.0
  creationTimestamp: null
  labels:
    cluster.x-k8s.io/provider: infrastructure-lxd
    cluster.x-k8s.io/v1beta1: v1alpha1
  name: lxdmachines.infrastructure.cluster.x-k8s.io
 spec:
  group: infrastructure.cluster.x-k8s.io
  names:
    categories:
    - cluster-api
    kind: LXDMachine
    listKind: LXDMachineList
    plural: lxdmachines
    shortNames:
    - lm
    singular: lxdmachine
  scope: Namespaced
  versions:
  - additionalPrinterColumns:
    - description: Cluster to which this LXDMachine belongs
      jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name
      name: Cluster
      type: string
    - description: LXD instance state
      jsonPath: .status.state
      name: State
      type: string
    - description: LXD instance ID
      jsonPath: .spec.providerID
      name: ProviderID
      type: string
    - description: Machine ready status
      jsonPath: .status.ready
      name: Ready
      type: string
    - description: Machine object which owns with this LXDMachine
      jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name
      name: Machine
      type: string
    - description: Time duration since creation of LXDMachine
      jsonPath: .metadata.creationTimestamp
      name: Age
      type: date
    name: v1alpha1
    schema:
      openAPIV3Schema:
        description: LXDMachine is the Schema for the lxdmachines API
        properties:
          apiVersion:
            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            description: LXDMachineSpec defines the desired state of LXDMachine
            properties:
              imageAlias:
                description: Image is the image alias name to use.
                type: string
              instanceType:
                description: InstanceType is the instance type to create.
                enum:
                - container
                - virtual-machine
                type: string
              profiles:
                description: Profiles is a list of profiles to attach to the instance.
                items:
                  type: string
                type: array
              providerID:
                description: ProviderID is the container name in ProviderID format (lxd:///<containername>)
                type: string
            type: object
          status:
            description: LXDMachineStatus defines the observed state of LXDMachine
            properties:
              addresses:
                items:
                  description: NodeAddress contains information for the node's address.
                  properties:
                    address:
                      description: The node address.
                      type: string
                    type:
                      description: Node address type, one of Hostname, ExternalIP or InternalIP.
                      type: string
                  required:
                  - address
                  - type
                  type: object
                type: array
              ready:
                type: boolean
              state:
                type: string
            required:
            - addresses
            type: object
        type: object
    served: true
    storage: true
    subresources:
      status: {}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.9.0
  creationTimestamp: null
  labels:
    cluster.x-k8s.io/provider: infrastructure-lxd
    cluster.x-k8s.io/v1beta1: v1alpha1
  name: lxdmachinetemplates.infrastructure.cluster.x-k8s.io
 spec:
  group: infrastructure.cluster.x-k8s.io
  names:
    kind: LXDMachineTemplate
    listKind: LXDMachineTemplateList
    plural: lxdmachinetemplates
    singular: lxdmachinetemplate
  scope: Namespaced
  versions:
  - name: v1alpha1
    schema:
      openAPIV3Schema:
        description: LXDMachineTemplate is the Schema for the lxdmachinetemplates API
        properties:
          apiVersion:
            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            description: LXDMachineTemplateSpec defines the desired state of LXDMachineTemplate
            properties:
              template:
                properties:
                  metadata:
                    description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
                    properties:
                      annotations:
                        additionalProperties:
                          type: string
                        description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
                        type: object
                      labels:
                        additionalProperties:
                          type: string
                        description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
                        type: object
                    type: object
                  spec:
                    description: Spec is the specification of the desired behaviour of the machine.
                    properties:
                      imageAlias:
                        description: Image is the image alias name to use.
                        type: string
                      instanceType:
                        description: InstanceType is the instance type to create.
                        enum:
                        - container
                        - virtual-machine
                        type: string
                      profiles:
                        description: Profiles is a list of profiles to attach to the instance.
                        items:
                          type: string
                        type: array
                      providerID:
                        description: ProviderID is the container name in ProviderID format (lxd:///<containername>)
                        type: string
                    type: object
                required:
                - spec
                type: object
            required:
            - template
            type: object
        type: object
    served: true
    storage: true
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: capl-controller-manager
  namespace: capl-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: capl-leader-election-role
  namespace: capl-system
 rules:
 - apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
 - apiGroups:
  - coordination.k8s.io
  resources:
  - leases
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
 - apiGroups:
  - ""
  resources:
  - events
  verbs:
  - create
  - patch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  creationTimestamp: null
  name: capl-manager-role
 rules:
 - apiGroups:
  - ""
  resources:
  - secrets
  verbs:
  - get
  - list
  - watch
 - apiGroups:
  - cluster.x-k8s.io
  resources:
  - clusters
  - clusters/status
  verbs:
  - get
  - list
  - watch
 - apiGroups:
  - cluster.x-k8s.io
  resources:
  - machines
  - machines/status
  verbs:
  - get
  - list
  - watch
 - apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdclusters
  verbs:
  - create
  - delete
  - get
  - list
  - patch
  - update
  - watch
 - apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdclusters/finalizers
  verbs:
  - update
 - apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdclusters/status
  verbs:
  - get
  - patch
  - update
 - apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdmachines
  verbs:
  - create
  - delete
  - get
  - list
  - patch
  - update
  - watch
 - apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdmachines/finalizers
  verbs:
  - update
 - apiGroups:
  - infrastructure.cluster.x-k8s.io
  resources:
  - lxdmachines/status
  verbs:
  - get
  - patch
  - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: capl-metrics-reader
 rules:
 - nonResourceURLs:
  - /metrics
  verbs:
  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: capl-proxy-role
 rules:
 - apiGroups:
  - authentication.k8s.io
  resources:
  - tokenreviews
  verbs:
  - create
 - apiGroups:
  - authorization.k8s.io
  resources:
  - subjectaccessreviews
  verbs:
  - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: capl-leader-election-rolebinding
  namespace: capl-system
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: capl-leader-election-role
 subjects:
 - kind: ServiceAccount
  name: capl-controller-manager
  namespace: capl-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: capl-manager-rolebinding
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: capl-manager-role
 subjects:
 - kind: ServiceAccount
  name: capl-controller-manager
  namespace: capl-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: capl-proxy-rolebinding
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: capl-proxy-role
 subjects:
 - kind: ServiceAccount
  name: capl-controller-manager
  namespace: capl-system
 ---
 apiVersion: v1
 data:
  controller_manager_config.yaml: "apiVersion: controller-runtime.sigs.k8s.io/v1alpha1\nkind: ControllerManagerConfig\nhealth:\n  healthProbeBindAddress: :8081\nmetrics:\n  bindAddress: 127.0.0.1:8080\nwebhook:\n  port: 9443\nleaderElection:\n  leaderElect: true\n  resourceName: 349154e5.cluster.x-k8s.io\n#   leaderElectionReleaseOnCancel defines if the leader should step down volume \n#   when the Manager ends. This requires the binary to immediately end when the\n#   Manager is stopped, otherwise, this setting is unsafe. Setting this significantly\n#   speeds up voluntary leader transitions as the new leader don't have to wait\n#   LeaseDuration time first.\n#   In the default scaffold provided, the program ends immediately after \n#   the manager stops, so would be fine to enable this option. However, \n#   if you are doing or is intended to do any operation such as perform cleanups \n#   after the manager stops then its usage might be unsafe.\n#   leaderElectionReleaseOnCancel: true\n"
 kind: ConfigMap
 metadata:
  name: capl-manager-config
  namespace: capl-system
 ---
 apiVersion: v1
 kind: Service
 metadata:
  labels:
    control-plane: controller-manager
  name: capl-controller-manager-metrics-service
  namespace: capl-system
 spec:
  ports:
  - name: https
    port: 8443
    protocol: TCP
    targetPort: https
  selector:
    control-plane: controller-manager
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  labels:
    control-plane: controller-manager
  name: capl-controller-manager
  namespace: capl-system
 spec:
  replicas: 1
  selector:
    matchLabels:
      control-plane: controller-manager
  template:
    metadata:
      annotations:
        kubectl.kubernetes.io/default-container: manager
      labels:
        control-plane: controller-manager
    spec:
      containers:
      - args:
        - --secure-listen-address=0.0.0.0:8443
        - --upstream=http://127.0.0.1:8080/
        - --logtostderr=true
        - --v=0
        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0
        name: kube-rbac-proxy
        ports:
        - containerPort: 8443
          name: https
          protocol: TCP
        resources:
          limits:
            cpu: 500m
            memory: 128Mi
          requests:
            cpu: 5m
            memory: 64Mi
        securityContext:
          allowPrivilegeEscalation: false
      - args:
        - --health-probe-bind-address=:8081
        - --metrics-bind-address=127.0.0.1:8080
        - --leader-elect
        command:
        - /manager
        envFrom:
        - configMapRef:
            name: lxd-socket
        image: neoaggelos/capi-lxd:dev1
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8081
          initialDelaySeconds: 15
          periodSeconds: 20
        name: manager
        readinessProbe:
          httpGet:
            path: /readyz
            port: 8081
          initialDelaySeconds: 5
          periodSeconds: 10
        resources:
          limits:
            cpu: 500m
            memory: 128Mi
          requests:
            cpu: 10m
            memory: 64Mi
        securityContext:
          allowPrivilegeEscalation: false
      securityContext:
        runAsNonRoot: true
      serviceAccountName: capl-controller-manager
      terminationGracePeriodSeconds: 10
	apiVersion: cluster.x-k8s.io/v1beta1
	kind: Cluster
	metadata:
	name: test
	namespace: default
	spec:
	controlPlaneRef:
	apiVersion: controlplane.cluster.x-k8s.io/v1beta1
	kind: MicroK8sControlPlane
	name: test-control-plane
	infrastructureRef:
	apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
	kind: LXDCluster
	name: test
	---
	apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
	kind: LXDCluster
	metadata:
	name: test
	namespace: default
	spec: {}
	---
	apiVersion: controlplane.cluster.x-k8s.io/v1beta1
	kind: MicroK8sControlPlane
	metadata:
	name: test-control-plane
	namespace: default
	spec:
	controlPlaneConfig:
	clusterConfiguration:
	portCompatibilityRemap: true
	initConfiguration:
	IPinIP: true
	addons:
	- dns
	- ingress
	joinTokenTTLInSecs: 9000
	machineTemplate:
	infrastructureTemplate:
	apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
	kind: LXDMachineTemplate
	name: test-control-plane
	replicas: 1
	version: v1.25.0
	---
	apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
	kind: LXDMachineTemplate
	metadata:
	name: test-control-plane
	namespace: default
	spec:
	template:
	spec:
	imageAlias: u22
	instanceType: container
	profiles:
	- default
	- microk8s
	---
	apiVersion: cluster.x-k8s.io/v1beta1
	kind: MachineDeployment
	metadata:
	name: test-md-0
	namespace: default
	spec:
	clusterName: test
	replicas: 0
	selector:
	matchLabels: null
	template:
	spec:
	bootstrap:
	configRef:
	apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
	kind: MicroK8sConfigTemplate
	name: test-md-0
	clusterName: test
	infrastructureRef:
	apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
	kind: LXDMachineTemplate
	name: test-md-0
	version: 1.25.0
	---
	apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
	kind: LXDMachineTemplate
	metadata:
	name: test-md-0
	namespace: default
	spec:
	template:
	spec:
	imageAlias: u22
	instanceType: container
	profiles:
	- default
	- microk8s
	---
	apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
	kind: MicroK8sConfigTemplate
	metadata:
	name: test-md-0
	namespace: default
	spec:
	template:
	spec: {}
	# configure LXD, and allow access over HTTPS.
	# note the IP address (in this case, 10.0.3.181) and the trust password.
	$ sudo lxd init
	Would you like to use LXD clustering? (yes/no) [default=no]: yes
	What IP address or DNS name should be used to reach this node? [default=10.0.3.181]:
	Are you joining an existing cluster? (yes/no) [default=no]:
	What name should be used to identify this node in the cluster? [default=test-ovn]:
	Setup password authentication on the cluster? (yes/no) [default=no]: yes
	Trust password for new clients:
	Again:
	....

	# create image alias for ubuntu 22.04
	$ sudo lxc launch ubuntu:22.04 t1
	$ sudo lxc image alias create u22 ubuntu:22.04
	$ sudo lxc rm t1 --force

	# create profile
	$ lxc profile create microk8s
	$ curl https://raw.githubusercontent.com/ubuntu/microk8s/master/tests/lxc/microk8s.profile \| lxc profile edit microk8s

	# deploy LXD provider and configure access to server (replace IP and password)
	$ microk8s kubectl apply -f provider.yaml
	$ microk8s kubectl create configmap -n capl-system lxd-socket --from-literal=LXD_SERVER=https://10.0.3.181:8443 --from-literal=LXD_PASSWORD=password

	# deploy cluster.yaml. initially, it has 1 control plane node and 0 workers
	$ microk8s kubectl apply -f cluster.yaml

	# wait for init node to come up, check with 'lxc list' and note its IP address
	$ lxc list
	+-------------------------------+---------+-----------------------------+------+-----------+-----------+
	\| NAME \| STATE \| IPV4 \| IPV6 \| TYPE \| SNAPSHOTS \|
	+-------------------------------+---------+-----------------------------+------+-----------+-----------+
	\| test-test-control-plane-pkstp \| RUNNING \| 10.0.0.187 (eth0) \| \| CONTAINER \| 0 \|
	+-------------------------------+---------+-----------------------------+------+-----------+-----------+

	# after deployment, fix `test-kubeconfig` secret to unblock the control plane provider
	$ microk8s kubectl edit cluster test # change control api endpoint to 10.0.0.187:6443
	$ microk8s kubectl edit lxdcluster test # change control api endpoint to 10.0.0.187:6443
	$ clusterctl get kubeconfig test > kubeconfig
	$ vim kubeconfig # change 'https://TODO:12345' to 'https://10.0.0.187:6443'
	$ cat kubeconfig \| base64 -w0
	$ microk8s kubectl edit secret test-kubeconfig # change value to the new base64 string

	# cluster is now ready to scale. edit 'cluster.yaml', change 'replicas: 0' and re-apply to deploy worker nodes
	apiVersion: v1
	kind: Namespace
	metadata:
	labels:
	control-plane: controller-manager
	name: capl-system
	---
	apiVersion: apiextensions.k8s.io/v1
	kind: CustomResourceDefinition
	metadata:
	annotations:
	controller-gen.kubebuilder.io/version: v0.9.0
	creationTimestamp: null
	labels:
	cluster.x-k8s.io/provider: infrastructure-lxd
	cluster.x-k8s.io/v1beta1: v1alpha1
	name: lxdclusters.infrastructure.cluster.x-k8s.io
	spec:
	group: infrastructure.cluster.x-k8s.io
	names:
	categories:
	- cluster-api
	kind: LXDCluster
	listKind: LXDClusterList
	plural: lxdclusters
	shortNames:
	- lc
	singular: lxdcluster
	scope: Namespaced
	versions:
	- additionalPrinterColumns:
	- description: Cluster to which this LXDCluster belongs
	jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name
	name: Cluster
	type: string
	- description: Cluster infrastructure is ready for LXD instances
	jsonPath: .status.ready
	name: Ready
	type: string
	- description: Time duration since creation of LXDCluster
	jsonPath: .metadata.creationTimestamp
	name: Age
	type: date
	name: v1alpha1
	schema:
	openAPIV3Schema:
	description: LXDCluster is the Schema for the lxdclusters API
	properties:
	apiVersion:
	description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
	type: string
	kind:
	description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
	type: string
	metadata:
	type: object
	spec:
	description: LXDClusterSpec defines the desired state of LXDCluster
	properties:
	controlPlaneEndpoint:
	description: ControlPlaneEndpoint represents the endpoint to communicate with the control plane.
	properties:
	host:
	description: The hostname on which the API server is serving.
	type: string
	port:
	description: The port on which the API server is serving.
	format: int32
	type: integer
	required:
	- host
	- port
	type: object
	type: object
	status:
	description: LXDClusterStatus defines the observed state of LXDCluster
	properties:
	ready:
	description: Ready denotes that the LXD cluster (infrastructure) is ready.
	type: boolean
	required:
	- ready
	type: object
	type: object
	served: true
	storage: true
	subresources:
	status: {}
	---
	apiVersion: apiextensions.k8s.io/v1
	kind: CustomResourceDefinition
	metadata:
	annotations:
	controller-gen.kubebuilder.io/version: v0.9.0
	creationTimestamp: null
	labels:
	cluster.x-k8s.io/provider: infrastructure-lxd
	cluster.x-k8s.io/v1beta1: v1alpha1
	name: lxdclustertemplates.infrastructure.cluster.x-k8s.io
	spec:
	group: infrastructure.cluster.x-k8s.io
	names:
	kind: LXDClusterTemplate
	listKind: LXDClusterTemplateList
	plural: lxdclustertemplates
	singular: lxdclustertemplate
	scope: Namespaced
	versions:
	- name: v1alpha1
	schema:
	openAPIV3Schema:
	description: LXDClusterTemplate is the Schema for the lxdclustertemplates API
	properties:
	apiVersion:
	description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
	type: string
	kind:
	description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
	type: string
	metadata:
	type: object
	spec:
	description: LXDClusterTemplateSpec defines the desired state of LXDClusterTemplate
	properties:
	template:
	properties:
	metadata:
	description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
	properties:
	annotations:
	additionalProperties:
	type: string
	description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
	type: object
	labels:
	additionalProperties:
	type: string
	description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
	type: object
	type: object
	spec:
	description: Spec is the specification of the desired behaviour of the cluster.
	properties:
	controlPlaneEndpoint:
	description: ControlPlaneEndpoint represents the endpoint to communicate with the control plane.
	properties:
	host:
	description: The hostname on which the API server is serving.
	type: string
	port:
	description: The port on which the API server is serving.
	format: int32
	type: integer
	required:
	- host
	- port
	type: object
	type: object
	required:
	- spec
	type: object
	required:
	- template
	type: object
	type: object
	served: true
	storage: true
	---
	apiVersion: apiextensions.k8s.io/v1
	kind: CustomResourceDefinition
	metadata:
	annotations:
	controller-gen.kubebuilder.io/version: v0.9.0
	creationTimestamp: null
	labels:
	cluster.x-k8s.io/provider: infrastructure-lxd
	cluster.x-k8s.io/v1beta1: v1alpha1
	name: lxdmachines.infrastructure.cluster.x-k8s.io
	spec:
	group: infrastructure.cluster.x-k8s.io
	names:
	categories:
	- cluster-api
	kind: LXDMachine
	listKind: LXDMachineList
	plural: lxdmachines
	shortNames:
	- lm
	singular: lxdmachine
	scope: Namespaced
	versions:
	- additionalPrinterColumns:
	- description: Cluster to which this LXDMachine belongs
	jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name
	name: Cluster
	type: string
	- description: LXD instance state
	jsonPath: .status.state
	name: State
	type: string
	- description: LXD instance ID
	jsonPath: .spec.providerID
	name: ProviderID
	type: string
	- description: Machine ready status
	jsonPath: .status.ready
	name: Ready
	type: string
	- description: Machine object which owns with this LXDMachine
	jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name
	name: Machine
	type: string
	- description: Time duration since creation of LXDMachine
	jsonPath: .metadata.creationTimestamp
	name: Age
	type: date
	name: v1alpha1
	schema:
	openAPIV3Schema:
	description: LXDMachine is the Schema for the lxdmachines API
	properties:
	apiVersion:
	description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
	type: string
	kind:
	description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
	type: string
	metadata:
	type: object
	spec:
	description: LXDMachineSpec defines the desired state of LXDMachine
	properties:
	imageAlias:
	description: Image is the image alias name to use.
	type: string
	instanceType:
	description: InstanceType is the instance type to create.
	enum:
	- container
	- virtual-machine
	type: string
	profiles:
	description: Profiles is a list of profiles to attach to the instance.
	items:
	type: string
	type: array
	providerID:
	description: ProviderID is the container name in ProviderID format (lxd:///<containername>)
	type: string
	type: object
	status:
	description: LXDMachineStatus defines the observed state of LXDMachine
	properties:
	addresses:
	items:
	description: NodeAddress contains information for the node's address.
	properties:
	address:
	description: The node address.
	type: string
	type:
	description: Node address type, one of Hostname, ExternalIP or InternalIP.
	type: string
	required:
	- address
	- type
	type: object
	type: array
	ready:
	type: boolean
	state:
	type: string
	required:
	- addresses
	type: object
	type: object
	served: true
	storage: true
	subresources:
	status: {}
	---
	apiVersion: apiextensions.k8s.io/v1
	kind: CustomResourceDefinition
	metadata:
	annotations:
	controller-gen.kubebuilder.io/version: v0.9.0
	creationTimestamp: null
	labels:
	cluster.x-k8s.io/provider: infrastructure-lxd
	cluster.x-k8s.io/v1beta1: v1alpha1
	name: lxdmachinetemplates.infrastructure.cluster.x-k8s.io
	spec:
	group: infrastructure.cluster.x-k8s.io
	names:
	kind: LXDMachineTemplate
	listKind: LXDMachineTemplateList
	plural: lxdmachinetemplates
	singular: lxdmachinetemplate
	scope: Namespaced
	versions:
	- name: v1alpha1
	schema:
	openAPIV3Schema:
	description: LXDMachineTemplate is the Schema for the lxdmachinetemplates API
	properties:
	apiVersion:
	description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
	type: string
	kind:
	description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
	type: string
	metadata:
	type: object
	spec:
	description: LXDMachineTemplateSpec defines the desired state of LXDMachineTemplate
	properties:
	template:
	properties:
	metadata:
	description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
	properties:
	annotations:
	additionalProperties:
	type: string
	description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
	type: object
	labels:
	additionalProperties:
	type: string
	description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
	type: object
	type: object
	spec:
	description: Spec is the specification of the desired behaviour of the machine.
	properties:
	imageAlias:
	description: Image is the image alias name to use.
	type: string
	instanceType:
	description: InstanceType is the instance type to create.
	enum:
	- container
	- virtual-machine
	type: string
	profiles:
	description: Profiles is a list of profiles to attach to the instance.
	items:
	type: string
	type: array
	providerID:
	description: ProviderID is the container name in ProviderID format (lxd:///<containername>)
	type: string
	type: object
	required:
	- spec
	type: object
	required:
	- template
	type: object
	type: object
	served: true
	storage: true
	---
	apiVersion: v1
	kind: ServiceAccount
	metadata:
	name: capl-controller-manager
	namespace: capl-system
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: Role
	metadata:
	name: capl-leader-election-role
	namespace: capl-system
	rules:
	- apiGroups:
	- ""
	resources:
	- configmaps
	verbs:
	- get
	- list
	- watch
	- create
	- update
	- patch
	- delete
	- apiGroups:
	- coordination.k8s.io
	resources:
	- leases
	verbs:
	- get
	- list
	- watch
	- create
	- update
	- patch
	- delete
	- apiGroups:
	- ""
	resources:
	- events
	verbs:
	- create
	- patch
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: ClusterRole
	metadata:
	creationTimestamp: null
	name: capl-manager-role
	rules:
	- apiGroups:
	- ""
	resources:
	- secrets
	verbs:
	- get
	- list
	- watch
	- apiGroups:
	- cluster.x-k8s.io
	resources:
	- clusters
	- clusters/status
	verbs:
	- get
	- list
	- watch
	- apiGroups:
	- cluster.x-k8s.io
	resources:
	- machines
	- machines/status
	verbs:
	- get
	- list
	- watch
	- apiGroups:
	- infrastructure.cluster.x-k8s.io
	resources:
	- lxdclusters
	verbs:
	- create
	- delete
	- get
	- list
	- patch
	- update
	- watch
	- apiGroups:
	- infrastructure.cluster.x-k8s.io
	resources:
	- lxdclusters/finalizers
	verbs:
	- update
	- apiGroups:
	- infrastructure.cluster.x-k8s.io
	resources:
	- lxdclusters/status
	verbs:
	- get
	- patch
	- update
	- apiGroups:
	- infrastructure.cluster.x-k8s.io
	resources:
	- lxdmachines
	verbs:
	- create
	- delete
	- get
	- list
	- patch
	- update
	- watch
	- apiGroups:
	- infrastructure.cluster.x-k8s.io
	resources:
	- lxdmachines/finalizers
	verbs:
	- update
	- apiGroups:
	- infrastructure.cluster.x-k8s.io
	resources:
	- lxdmachines/status
	verbs:
	- get
	- patch
	- update
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: ClusterRole
	metadata:
	name: capl-metrics-reader
	rules:
	- nonResourceURLs:
	- /metrics
	verbs:
	- get
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: ClusterRole
	metadata:
	name: capl-proxy-role
	rules:
	- apiGroups:
	- authentication.k8s.io
	resources:
	- tokenreviews
	verbs:
	- create
	- apiGroups:
	- authorization.k8s.io
	resources:
	- subjectaccessreviews
	verbs:
	- create
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: RoleBinding
	metadata:
	name: capl-leader-election-rolebinding
	namespace: capl-system
	roleRef:
	apiGroup: rbac.authorization.k8s.io
	kind: Role
	name: capl-leader-election-role
	subjects:
	- kind: ServiceAccount
	name: capl-controller-manager
	namespace: capl-system
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: ClusterRoleBinding
	metadata:
	name: capl-manager-rolebinding
	roleRef:
	apiGroup: rbac.authorization.k8s.io
	kind: ClusterRole
	name: capl-manager-role
	subjects:
	- kind: ServiceAccount
	name: capl-controller-manager
	namespace: capl-system
	---
	apiVersion: rbac.authorization.k8s.io/v1
	kind: ClusterRoleBinding
	metadata:
	name: capl-proxy-rolebinding
	roleRef:
	apiGroup: rbac.authorization.k8s.io
	kind: ClusterRole
	name: capl-proxy-role
	subjects:
	- kind: ServiceAccount
	name: capl-controller-manager
	namespace: capl-system
	---
	apiVersion: v1
	data:
	controller_manager_config.yaml: "apiVersion: controller-runtime.sigs.k8s.io/v1alpha1\nkind: ControllerManagerConfig\nhealth:\n healthProbeBindAddress: :8081\nmetrics:\n bindAddress: 127.0.0.1:8080\nwebhook:\n port: 9443\nleaderElection:\n leaderElect: true\n resourceName: 349154e5.cluster.x-k8s.io\n# leaderElectionReleaseOnCancel defines if the leader should step down volume \n# when the Manager ends. This requires the binary to immediately end when the\n# Manager is stopped, otherwise, this setting is unsafe. Setting this significantly\n# speeds up voluntary leader transitions as the new leader don't have to wait\n# LeaseDuration time first.\n# In the default scaffold provided, the program ends immediately after \n# the manager stops, so would be fine to enable this option. However, \n# if you are doing or is intended to do any operation such as perform cleanups \n# after the manager stops then its usage might be unsafe.\n# leaderElectionReleaseOnCancel: true\n"
	kind: ConfigMap
	metadata:
	name: capl-manager-config
	namespace: capl-system
	---
	apiVersion: v1
	kind: Service
	metadata:
	labels:
	control-plane: controller-manager
	name: capl-controller-manager-metrics-service
	namespace: capl-system
	spec:
	ports:
	- name: https
	port: 8443
	protocol: TCP
	targetPort: https
	selector:
	control-plane: controller-manager
	---
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	labels:
	control-plane: controller-manager
	name: capl-controller-manager
	namespace: capl-system
	spec:
	replicas: 1
	selector:
	matchLabels:
	control-plane: controller-manager
	template:
	metadata:
	annotations:
	kubectl.kubernetes.io/default-container: manager
	labels:
	control-plane: controller-manager
	spec:
	containers:
	- args:
	- --secure-listen-address=0.0.0.0:8443
	- --upstream=http://127.0.0.1:8080/
	- --logtostderr=true
	- --v=0
	image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0
	name: kube-rbac-proxy
	ports:
	- containerPort: 8443
	name: https
	protocol: TCP
	resources:
	limits:
	cpu: 500m
	memory: 128Mi
	requests:
	cpu: 5m
	memory: 64Mi
	securityContext:
	allowPrivilegeEscalation: false
	- args:
	- --health-probe-bind-address=:8081
	- --metrics-bind-address=127.0.0.1:8080
	- --leader-elect
	command:
	- /manager
	envFrom:
	- configMapRef:
	name: lxd-socket
	image: neoaggelos/capi-lxd:dev1
	livenessProbe:
	httpGet:
	path: /healthz
	port: 8081
	initialDelaySeconds: 15
	periodSeconds: 20
	name: manager
	readinessProbe:
	httpGet:
	path: /readyz
	port: 8081
	initialDelaySeconds: 5
	periodSeconds: 10
	resources:
	limits:
	cpu: 500m
	memory: 128Mi
	requests:
	cpu: 10m
	memory: 64Mi
	securityContext:
	allowPrivilegeEscalation: false
	securityContext:
	runAsNonRoot: true
	serviceAccountName: capl-controller-manager
	terminationGracePeriodSeconds: 10