Patch versions updates (#12782 )

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Patch versions updates (#12763 )
2026-06-02 09:37:58 +00:00 · 2025-12-11 00:55:31 -08:00 · 2025-12-04 06:39:01 -08:00 · 2025-11-25 00:52:36 -08:00 · 2025-11-19 23:32:02 -08:00 · 2025-11-18 20:09:59 -08:00
311 changed files with 5688 additions and 2889 deletions
@@ -1,4 +1,5 @@
 ---
+parseable: true
 skip_list:
  # see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules

@@ -33,8 +34,6 @@ skip_list:
  # Disable run-once check with free strategy
  # (Disabled in June 2023 after ansible upgrade; FIXME)
  - 'run-once[task]'
-
-  - 'jinja[spacing]'
 exclude_paths:
  # Generated files
  - tests/files/custom_cni/cilium.yaml
@@ -13,16 +13,16 @@ jobs:
      issues: write

    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8

      - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@cb6e97157cbf851e3a393ff8d57c93a484cc323f
+        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
        id: issue-parser
        with:
          template-path: .github/ISSUE_TEMPLATE/bug-report.yaml

      - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@b80ae64e3e156e9c111b075bfa04b295d54e8e2e
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@e38e6809c5420d038eed380d49ee9a6ca7c92dbf
        with:
          issue-form: ${{ steps.issue-parser.outputs.jsonString }}
          section: os
@@ -13,14 +13,14 @@ jobs:
    outputs:
      branches: ${{ steps.get-branches.outputs.data }}
    steps:
-    - uses: octokit/graphql-action@ddde8ebb2493e79f390e6449c725c21663a67505
+    - uses: octokit/graphql-action@8ad880e4d437783ea2ab17010324de1075228110
      id: get-branches
      with:
        query: |
          query get_release_branches($owner:String!, $name:String!) {
            repository(owner:$owner, name:$name) {
              refs(refPrefix: "refs/heads/",
-                   first: 3,
+                   first: 1, # TODO increment once we have release branch with the new checksums format
                   query: "release-",
                   orderBy: {
                     field: ALPHABETICAL,
@@ -11,10 +11,10 @@ jobs:
  update-patch-versions:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
      with:
        ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+    - uses: actions/setup-python@v6
      with:
        python-version: '3.13'
        cache: 'pip'
@@ -22,14 +22,14 @@ jobs:
    - run: update-hashes
      env:
        API_KEY: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+    - uses: actions/cache@v4
      with:
        key: pre-commit-hook-propagate
        path: |
          ~/.cache/pre-commit
    - run: pre-commit run --all-files propagate-ansible-variables
      continue-on-error: true
-    - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1
+    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
      with:
        commit-message: Patch versions updates
        title: Patch versions updates - ${{ inputs.branch }}
@@ -40,37 +40,20 @@ pr:
          - debian11-macvlan
          - debian12-cilium
          - debian13-cilium
-          - fedora42-kube-router
-          - fedora43-kube-router
-          - fedora42-calico
+          - fedora39-kube-router
+          - openeuler24-calico
          - rockylinux9-cilium
-          - rockylinux10-cilium
          - ubuntu22-calico-all-in-one
          - ubuntu22-calico-all-in-one-upgrade
          - ubuntu24-calico-etcd-datastore
          - ubuntu24-calico-all-in-one-hardening
          - ubuntu24-cilium-sep
-          - ubuntu24-crio-scale
-          - ubuntu24-crio-upgrade
          - ubuntu24-flannel-collection
          - ubuntu24-kube-router-sep
          - ubuntu24-kube-router-svc-proxy
          - ubuntu24-ha-separate-etcd
-          - fedora43-flannel-crio-collection-scale
-
-# This is for flakey test so they don't disrupt the PR worklflow too much.
-# Jobs here MUST have a open issue so we don't lose sight of them
-pr-flakey:
-  extends: pr
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-  retry: 2
-  parallel:
-    matrix:
-      - TESTCASE:
-          - flatcar4081-calico # https://github.com/kubernetes-sigs/kubespray/issues/12309
-          - openeuler24-calico # https://github.com/kubernetes-sigs/kubespray/issues/12877
+          - flatcar4081-calico
+          - fedora40-flannel-crio-collection-scale

 # The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
 ubuntu24-calico-all-in-one:
@@ -105,10 +88,8 @@ pr_full:
          - debian11-custom-cni
          - debian11-kubelet-csr-approver
          - debian12-custom-cni-helm
-          - fedora42-calico-swap-selinux
-          - fedora42-crio
-          - fedora43-calico-swap-selinux
-          - fedora43-crio
+          - fedora39-calico-swap-selinux
+          - fedora39-crio
          - ubuntu24-calico-ha-wireguard
          - ubuntu24-flannel-ha
          - ubuntu24-flannel-ha-once
@@ -146,7 +127,6 @@ pr_extended:
          - debian12-docker
          - debian13-calico
          - rockylinux9-calico
-          - rockylinux10-calico
          - ubuntu22-all-in-one-docker
          - ubuntu24-all-in-one-docker
          - ubuntu24-calico-all-in-one
@@ -166,9 +146,8 @@ periodic:
          - debian11-calico-upgrade
          - debian11-calico-upgrade-once
          - debian12-cilium-svc-proxy
-          - fedora42-calico-selinux
-          - fedora43-calico-selinux
-          - fedora43-docker-calico
+          - fedora39-calico-selinux
+          - fedora40-docker-calico
          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
          - ubuntu24-calico-ha-recover
          - ubuntu24-calico-ha-recover-noquorum
@@ -35,6 +35,7 @@ molecule:
      - container-engine/containerd
      - container-engine/cri-o
      - container-engine/gvisor
+      - container-engine/youki
      - adduser
      - bastion-ssh-config
      - bootstrap_os
@@ -52,5 +53,3 @@ molecule_full:
    - ROLE:
      # FIXME : tests below are perma-failing
      - container-engine/kata-containers
-      # FIXME: until youki release 0.6.1
-      - container-engine/youki
@@ -37,6 +37,7 @@ terraform_validate:
          - hetzner
          - vsphere
          - upcloud
+          - nifcloud

 .terraform_apply:
  extends: .terraform_install
@@ -88,10 +89,11 @@ tf-elastx_cleanup:
    - ./scripts/openstack-cleanup/main.py
  allow_failure: true

-tf-elastx_ubuntu24-calico:
+tf-elastx_ubuntu20-calico:
  extends: .terraform_apply
  stage: deploy-part1
  when: on_success
+  allow_failure: true
  variables:
    <<: *elastx_variables
    PROVIDER: openstack
@@ -114,6 +116,5 @@ tf-elastx_ubuntu24-calico:
    TF_VAR_az_list_node: '["sto1"]'
    TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b    # v1-standard-2
    TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
-    TF_VAR_image: ubuntu-24.04-server-latest
+    TF_VAR_image: ubuntu-20.04-server-latest
    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
-    TESTCASE: $CI_JOB_NAME
@@ -36,7 +36,7 @@ vagrant:
    policy: pull-push # TODO: change to "pull" when not on main
  stage: deploy-extended
  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
      when: on_success
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
      when: on_success
@@ -33,7 +33,6 @@ repos:
    hooks:
      - id: ansible-lint
        additional_dependencies:
-          - ansible-core>=2.18.0,<2.19.0
          - jmespath==1.0.1
          - netaddr==1.3.0
          - distlib
@@ -50,7 +49,7 @@ repos:
        name: Build and install kubernetes-sigs.kubespray Ansible collection
        language: python
        additional_dependencies:
-          - ansible-core>=2.18.0,<2.19.0
+          - ansible-core>=2.16.4
          - distlib
        entry: tests/scripts/collection-build-install.sh
        pass_filenames: false
@@ -92,7 +91,7 @@ repos:
        name: Update static files referencing default kubespray values
        language: python
        additional_dependencies:
-          - ansible-core>=2.18.0,<2.19.0
+          - ansible-core>=2.16.4
        entry: scripts/propagate_ansible_variables.yml
        pass_filenames: false

@@ -12,6 +12,7 @@ To install development dependencies you can set up a python virtual env with the
 virtualenv venv
 source venv/bin/activate
 pip install -r tests/requirements.txt
+ansible-galaxy install -r tests/requirements.yml
 ```

 #### Linting
@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1

-# Use immutable image tags rather than mutable tags (like ubuntu:24.04)
-FROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b
+# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
+FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37

 # Some tools like yamllint need this
 # Pip needs this as well at the moment to install ansible
@@ -29,14 +29,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \

 RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
    --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \
-    pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt \
+    pip install --no-compile --no-cache-dir -r requirements.txt \
    && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \;

 SHELL ["/bin/bash", "-o", "pipefail", "-c"]

 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.36.1/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.36.1/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
    && chmod a+x /usr/local/bin/kubectl

 COPY *.yml ./
@@ -8,7 +8,6 @@ aliases:
  kubespray-reviewers:
    - cyclinder
    - erikjiang
-    - guoard
    - mzaian
    - tico88612
    - vannten
@@ -22,7 +22,7 @@ Ensure you have installed Docker then
 ```ShellSession
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.31.0 bash
+  quay.io/kubespray/kubespray:v2.29.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -89,13 +89,13 @@ vagrant up
 - **Flatcar Container Linux by Kinvolk**
 - **Debian** Bookworm, Bullseye, Trixie
 - **Ubuntu** 22.04, 24.04
- **CentOS Stream / RHEL** 9, 10
- **Fedora** 39, 40, 41, 42
+- **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
+- **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
- **Oracle Linux** 9, 10
- **Alma Linux** 9, 10
- **Rocky Linux** 9, 10 (experimental in 10: see [Rocky Linux 10 notes](docs/operating_systems/rhel.md#rocky-linux-10))
+- **Oracle Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
+- **Alma Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
+- **Rocky Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
 - **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
 - **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
@@ -111,23 +111,24 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->

 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.36.1
-  - [etcd](https://github.com/etcd-io/etcd) 3.6.11
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.33.7
+  - [etcd](https://github.com/etcd-io/etcd) 3.5.25
  - [docker](https://www.docker.com/) 28.3
-  - [containerd](https://containerd.io/) 2.3.1
-  - [cri-o](https://cri-o.io/) 1.36.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [containerd](https://containerd.io/) 2.1.5
+  - [cri-o](http://cri-o.io/) 1.33.7 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.9.1
-  - [calico](https://github.com/projectcalico/calico) 3.31.5
-  - [cilium](https://github.com/cilium/cilium) 1.19.4
-  - [flannel](https://github.com/flannel-io/flannel) 0.28.4
+  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
+  - [calico](https://github.com/projectcalico/calico) 3.30.5
+  - [cilium](https://github.com/cilium/cilium) 1.18.4
+  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
  - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
  - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
-  - [kube-vip](https://github.com/kube-vip/kube-vip) 1.0.3
+  - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
 - Application
  - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.14.2
+  - [coredns](https://github.com/coredns/coredns) 1.12.0
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
  - [argocd](https://argoproj.github.io/) 2.14.5
  - [helm](https://helm.sh/) 3.18.4
  - [metallb](https://metallb.universe.tf/) 0.13.9
@@ -201,6 +202,8 @@ See also [Network checker](docs/advanced/netcheck.md).

 ## Ingress Plugins

+- [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller.
+
 - [metallb](docs/ingress/metallb.md): the MetalLB bare-metal service LoadBalancer provider.

 ## Community docs and resources
@@ -15,7 +15,7 @@ The Kubespray Project is released on an as-needed basis. The process is as follo
 1. The release issue is closed
 1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
 1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
-1. Create/Update Issue for upgrading kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)
+1. Create/Update Issue for upgradeing kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)

 ## Major/minor releases and milestones

@@ -58,7 +58,7 @@ You can create a release note with:
 export GITHUB_TOKEN=<your-github-token>
 export ORG=kubernetes-sigs
 export REPO=kubespray
-release-notes generate --org "${ORG}" --repo "${REPO}" --repo-path "${PWD}" --start-sha <The start commit-id> --end-sha <The end commit-id> --dependencies=false --output=/tmp/kubespray-release-note
+release-notes --start-sha <The start commit-id> --end-sha <The end commit-id> --dependencies=false --output=/tmp/kubespray-release-note --required-author=""
 ```

 If the release note file(/tmp/kubespray-release-note) contains "### Uncategorized" pull requests, those pull requests don't have a valid kind label(`kind/feature`, etc.).
@@ -35,9 +35,6 @@ SUPPORTED_OS = {
  "fedora40"            => {box: "fedora/40-cloud-base",       user: "vagrant"},
  "fedora39-arm64"      => {box: "bento/fedora-39-arm64",      user: "vagrant"},
  "fedora40-arm64"      => {box: "bento/fedora-40",            user: "vagrant"},
-  "fedora41"            => {box: "fedora/41-cloud-base",       user: "vagrant"},
-  "fedora42"            => {box: "fedora/42-cloud-base",       user: "vagrant"},
-  "fedora41-bento"      => {box: "bento/fedora-41",            user: "vagrant"},
  "opensuse"            => {box: "opensuse/Leap-15.6.x86_64",  user: "vagrant"},
  "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
  "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
@@ -1,9 +0,0 @@
-#!/bin/bash -eux
-# Install collection from source assuming dependencies are present.
-# Run in SemaphoreUI this bash script can install Kubespray from the repo
-NAMESPACE=kubernetes_sigs
-COLLECTION=kubespray
-MY_VER=$(grep '^version:' galaxy.yml|cut -d: -f2|sed 's/ //')
-
-ansible-galaxy collection build --force --output-path .
-ansible-galaxy collection install --offline --force $NAMESPACE-$COLLECTION-$MY_VER.tar.gz
@@ -20,6 +20,7 @@ function create_container_image_tar() {

 		kubectl describe cronjobs,jobs,pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq > "${IMAGES}"
 		# NOTE: etcd and pause cannot be seen as pods.
+		# The pause image is used for --pod-infra-container-image option of kubelet.
 		kubectl cluster-info dump | grep -E "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g >> "${IMAGES}"
 	else
 		echo "Getting images from file \"${IMAGES_FROM_FILE}\""
@@ -51,7 +51,7 @@ To generate kubespray inventory based on the terraform state file you can run th
 You should now have a inventory file named `inventory.ini` that you can use with kubespray, e.g.

 ```bash
-ansible-playbook -i contrib/terraform/gcp/inventory.ini cluster.yml -b -v
+ansible-playbook -i contrib/terraform/gcs/inventory.ini cluster.yml -b -v
 ```

 ## Variables
@@ -0,0 +1,5 @@
+*.tfstate*
+.terraform.lock.hcl
+.terraform
+
+sample-inventory/inventory.ini
@@ -0,0 +1,138 @@
+# Kubernetes on NIFCLOUD with Terraform
+
+Provision a Kubernetes cluster on [NIFCLOUD](https://pfs.nifcloud.com/) using Terraform and Kubespray
+
+## Overview
+
+The setup looks like following
+
+```text
+                              Kubernetes cluster
+                        +----------------------------+
+---------------+       |   +--------------------+   |
+|               |       |   | +--------------------+ |
+| API server LB +---------> | |                    | |
+|               |       |   | | Control Plane/etcd | |
+---------------+       |   | | node(s)            | |
+                        |   +-+                    | |
+                        |     +--------------------+ |
+                        |           ^                |
+                        |           |                |
+                        |           v                |
+                        |   +--------------------+   |
+                        |   | +--------------------+ |
+                        |   | |                    | |
+                        |   | |        Worker      | |
+                        |   | |        node(s)     | |
+                        |   +-+                    | |
+                        |     +--------------------+ |
+                        +----------------------------+
+```
+
+## Requirements
+
+* Terraform 1.3.7
+
+## Quickstart
+
+### Export Variables
+
+* Your NIFCLOUD credentials:
+
+  ```bash
+  export NIFCLOUD_ACCESS_KEY_ID=<YOUR ACCESS KEY>
+  export NIFCLOUD_SECRET_ACCESS_KEY=<YOUR SECRET ACCESS KEY>
+  ```
+
+* The SSH KEY used to connect to the instance:
+  * FYI: [Cloud Help(SSH Key)](https://pfs.nifcloud.com/help/ssh.htm)
+
+  ```bash
+  export TF_VAR_SSHKEY_NAME=<YOUR SSHKEY NAME>
+  ```
+
+* The IP address to connect to bastion server:
+
+  ```bash
+  export TF_VAR_working_instance_ip=$(curl ifconfig.me)
+  ```
+
+### Create The Infrastructure
+
+* Run terraform:
+
+  ```bash
+  terraform init
+  terraform apply -var-file ./sample-inventory/cluster.tfvars
+  ```
+
+### Setup The Kubernetes
+
+* Generate cluster configuration file:
+
+  ```bash
+  ./generate-inventory.sh > sample-inventory/inventory.ini
+  ```
+
+* Export Variables:
+
+  ```bash
+  BASTION_IP=$(terraform output -json | jq -r '.kubernetes_cluster.value.bastion_info | to_entries[].value.public_ip')
+  API_LB_IP=$(terraform output -json | jq -r '.kubernetes_cluster.value.control_plane_lb')
+  CP01_IP=$(terraform output -json | jq -r '.kubernetes_cluster.value.control_plane_info | to_entries[0].value.private_ip')
+  export ANSIBLE_SSH_ARGS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand=\"ssh root@${BASTION_IP} -W %h:%p\""
+  ```
+
+* Set ssh-agent"
+
+  ```bash
+  eval `ssh-agent`
+  ssh-add <THE PATH TO YOUR SSH KEY>
+  ```
+
+* Run cluster.yml playbook:
+
+  ```bash
+  cd ./../../../
+  ansible-playbook -i contrib/terraform/nifcloud/inventory/inventory.ini cluster.yml
+  ```
+
+### Connecting to Kubernetes
+
+* [Install kubectl](https://kubernetes.io/docs/tasks/tools/) on the localhost
+* Fetching kubeconfig file:
+
+  ```bash
+  mkdir -p ~/.kube
+  scp -o ProxyCommand="ssh root@${BASTION_IP} -W %h:%p" root@${CP01_IP}:/etc/kubernetes/admin.conf ~/.kube/config
+  ```
+
+* Rewrite /etc/hosts
+
+  ```bash
+  sudo echo "${API_LB_IP} lb-apiserver.kubernetes.local" >> /etc/hosts
+  ```
+
+* Run kubectl
+
+  ```bash
+  kubectl get node
+  ```
+
+## Variables
+
+* `region`: Region where to run the cluster
+* `az`: Availability zone where to run the cluster
+* `private_ip_bn`: Private ip address of bastion server
+* `private_network_cidr`:  Subnet of private network
+* `instances_cp`: Machine to provision as Control Plane. Key of this object will be used as part of the machine' name
+  * `private_ip`: private ip address of machine
+* `instances_wk`: Machine to provision as Worker Node. Key of this object will be used as part of the machine' name
+  * `private_ip`: private ip address of machine
+* `instance_key_name`: The key name of the Key Pair to use for the instance
+* `instance_type_bn`: The instance type of bastion server
+* `instance_type_wk`: The instance type of worker node
+* `instance_type_cp`: The instance type of control plane
+* `image_name`: OS image used for the instance
+* `working_instance_ip`: The IP address to connect to bastion server
+* `accounting_type`: Accounting type. (1: monthly, 2: pay per use)
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+#
+# Generates a inventory file based on the terraform output.
+# After provisioning a cluster, simply run this command and supply the terraform state file
+# Default state file is terraform.tfstate
+#
+
+set -e
+
+TF_OUT=$(terraform output -json)
+
+CONTROL_PLANES=$(jq -r '.kubernetes_cluster.value.control_plane_info | to_entries[]'  <(echo "${TF_OUT}"))
+WORKERS=$(jq -r '.kubernetes_cluster.value.worker_info | to_entries[]'  <(echo "${TF_OUT}"))
+mapfile -t CONTROL_PLANE_NAMES < <(jq -r '.key'  <(echo "${CONTROL_PLANES}"))
+mapfile -t WORKER_NAMES < <(jq -r '.key'  <(echo "${WORKERS}"))
+
+API_LB=$(jq -r '.kubernetes_cluster.value.control_plane_lb' <(echo "${TF_OUT}"))
+
+echo "[all]"
+# Generate control plane hosts
+i=1
+for name in "${CONTROL_PLANE_NAMES[@]}"; do
+  private_ip=$(jq -r '. | select( .key=='"\"${name}\""' ) | .value.private_ip'  <(echo "${CONTROL_PLANES}"))
+  echo "${name} ansible_user=root ansible_host=${private_ip} access_ip=${private_ip} ip=${private_ip} etcd_member_name=etcd${i}"
+  i=$(( i + 1 ))
+done
+
+# Generate worker hosts
+for name in "${WORKER_NAMES[@]}"; do
+  private_ip=$(jq -r '. | select( .key=='"\"${name}\""' ) | .value.private_ip'  <(echo "${WORKERS}"))
+  echo "${name} ansible_user=root ansible_host=${private_ip} access_ip=${private_ip} ip=${private_ip}"
+done
+
+API_LB=$(jq -r '.kubernetes_cluster.value.control_plane_lb'  <(echo "${TF_OUT}"))
+
+echo ""
+echo "[all:vars]"
+echo "upstream_dns_servers=['8.8.8.8','8.8.4.4']"
+echo "loadbalancer_apiserver={'address':'${API_LB}','port':'6443'}"
+
+
+echo ""
+echo "[kube_control_plane]"
+for name in "${CONTROL_PLANE_NAMES[@]}"; do
+  echo "${name}"
+done
+
+echo ""
+echo "[etcd]"
+for name in "${CONTROL_PLANE_NAMES[@]}"; do
+  echo "${name}"
+done
+
+echo ""
+echo "[kube_node]"
+for name in "${WORKER_NAMES[@]}"; do
+  echo "${name}"
+done
+
+echo ""
+echo "[k8s_cluster:children]"
+echo "kube_control_plane"
+echo "kube_node"
@@ -0,0 +1,36 @@
+provider "nifcloud" {
+  region = var.region
+}
+
+module "kubernetes_cluster" {
+  source = "./modules/kubernetes-cluster"
+
+  availability_zone = var.az
+  prefix            = "dev"
+
+  private_network_cidr = var.private_network_cidr
+
+  instance_key_name = var.instance_key_name
+  instances_cp      = var.instances_cp
+  instances_wk      = var.instances_wk
+  image_name        = var.image_name
+
+  instance_type_bn = var.instance_type_bn
+  instance_type_cp = var.instance_type_cp
+  instance_type_wk = var.instance_type_wk
+
+  private_ip_bn = var.private_ip_bn
+
+  additional_lb_filter = [var.working_instance_ip]
+}
+
+resource "nifcloud_security_group_rule" "ssh_from_bastion" {
+  security_group_names = [
+    module.kubernetes_cluster.security_group_name.bastion
+  ]
+  type      = "IN"
+  from_port = 22
+  to_port   = 22
+  protocol  = "TCP"
+  cidr_ip   = var.working_instance_ip
+}
@@ -0,0 +1,301 @@
+#################################################
+##
+## Local variables
+##
+locals {
+  # e.g. east-11 is 11
+  az_num = reverse(split("-", var.availability_zone))[0]
+  # e.g. east-11 is e11
+  az_short_name = "${substr(reverse(split("-", var.availability_zone))[1], 0, 1)}${local.az_num}"
+
+  # Port used by the protocol
+  port_ssh     = 22
+  port_kubectl = 6443
+  port_kubelet = 10250
+
+  # calico: https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements#network-requirements
+  port_bgp   = 179
+  port_vxlan = 4789
+  port_etcd  = 2379
+}
+
+#################################################
+##
+## General
+##
+
+# data
+data "nifcloud_image" "this" {
+  image_name = var.image_name
+}
+
+# private lan
+resource "nifcloud_private_lan" "this" {
+  private_lan_name  = "${var.prefix}lan"
+  availability_zone = var.availability_zone
+  cidr_block        = var.private_network_cidr
+  accounting_type   = var.accounting_type
+}
+
+#################################################
+##
+## Bastion
+##
+resource "nifcloud_security_group" "bn" {
+  group_name        = "${var.prefix}bn"
+  description       = "${var.prefix} bastion"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "bn" {
+
+  instance_id    = "${local.az_short_name}${var.prefix}bn01"
+  security_group = nifcloud_security_group.bn.group_name
+  instance_type  = var.instance_type_bn
+
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = var.private_ip_bn
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}bn01"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+#################################################
+##
+## Control Plane
+##
+resource "nifcloud_security_group" "cp" {
+  group_name        = "${var.prefix}cp"
+  description       = "${var.prefix} control plane"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "cp" {
+  for_each = var.instances_cp
+
+  instance_id    = "${local.az_short_name}${var.prefix}${each.key}"
+  security_group = nifcloud_security_group.cp.group_name
+  instance_type  = var.instance_type_cp
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = each.value.private_ip
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}${each.key}"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+resource "nifcloud_load_balancer" "this" {
+  load_balancer_name = "${local.az_short_name}${var.prefix}cp"
+  accounting_type    = var.accounting_type
+  balancing_type     = 1 // Round-Robin
+  load_balancer_port = local.port_kubectl
+  instance_port      = local.port_kubectl
+  instances          = [for v in nifcloud_instance.cp : v.instance_id]
+  filter = concat(
+    [for k, v in nifcloud_instance.cp : v.public_ip],
+    [for k, v in nifcloud_instance.wk : v.public_ip],
+    var.additional_lb_filter,
+  )
+  filter_type = 1 // Allow
+}
+
+#################################################
+##
+## Worker
+##
+resource "nifcloud_security_group" "wk" {
+  group_name        = "${var.prefix}wk"
+  description       = "${var.prefix} worker"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "wk" {
+  for_each = var.instances_wk
+
+  instance_id    = "${local.az_short_name}${var.prefix}${each.key}"
+  security_group = nifcloud_security_group.wk.group_name
+  instance_type  = var.instance_type_wk
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = each.value.private_ip
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}${each.key}"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+#################################################
+##
+## Security Group Rule: Kubernetes
+##
+
+# ssh
+resource "nifcloud_security_group_rule" "ssh_from_bastion" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_ssh
+  to_port                    = local.port_ssh
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.bn.group_name
+}
+
+# kubectl
+resource "nifcloud_security_group_rule" "kubectl_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubectl
+  to_port                    = local.port_kubectl
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# kubelet
+resource "nifcloud_security_group_rule" "kubelet_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubelet
+  to_port                    = local.port_kubelet
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+resource "nifcloud_security_group_rule" "kubelet_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubelet
+  to_port                    = local.port_kubelet
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+#################################################
+##
+## Security Group Rule: calico
+##
+
+# vslan
+resource "nifcloud_security_group_rule" "vxlan_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_vxlan
+  to_port                    = local.port_vxlan
+  protocol                   = "UDP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+resource "nifcloud_security_group_rule" "vxlan_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_vxlan
+  to_port                    = local.port_vxlan
+  protocol                   = "UDP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# bgp
+resource "nifcloud_security_group_rule" "bgp_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_bgp
+  to_port                    = local.port_bgp
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+resource "nifcloud_security_group_rule" "bgp_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_bgp
+  to_port                    = local.port_bgp
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# etcd
+resource "nifcloud_security_group_rule" "etcd_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_etcd
+  to_port                    = local.port_etcd
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
@@ -0,0 +1,48 @@
+output "control_plane_lb" {
+  description = "The DNS name of LB for control plane"
+  value       = nifcloud_load_balancer.this.dns_name
+}
+
+output "security_group_name" {
+  description = "The security group used in the cluster"
+  value = {
+    bastion       = nifcloud_security_group.bn.group_name,
+    control_plane = nifcloud_security_group.cp.group_name,
+    worker        = nifcloud_security_group.wk.group_name,
+  }
+}
+
+output "private_network_id" {
+  description = "The private network used in the cluster"
+  value       = nifcloud_private_lan.this.id
+}
+
+output "bastion_info" {
+  description = "The basion information in cluster"
+  value = { (nifcloud_instance.bn.instance_id) : {
+    instance_id = nifcloud_instance.bn.instance_id,
+    unique_id   = nifcloud_instance.bn.unique_id,
+    private_ip  = nifcloud_instance.bn.private_ip,
+    public_ip   = nifcloud_instance.bn.public_ip,
+  } }
+}
+
+output "worker_info" {
+  description = "The worker information in cluster"
+  value = { for v in nifcloud_instance.wk : v.instance_id => {
+    instance_id = v.instance_id,
+    unique_id   = v.unique_id,
+    private_ip  = v.private_ip,
+    public_ip   = v.public_ip,
+  } }
+}
+
+output "control_plane_info" {
+  description = "The control plane information in cluster"
+  value = { for v in nifcloud_instance.cp : v.instance_id => {
+    instance_id = v.instance_id,
+    unique_id   = v.unique_id,
+    private_ip  = v.private_ip,
+    public_ip   = v.public_ip,
+  } }
+}
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#################################################
+##
+## IP Address
+##
+configure_private_ip_address () {
+  cat << EOS > /etc/netplan/01-netcfg.yaml
+network:
+    version: 2
+    renderer: networkd
+    ethernets:
+        ens192:
+            dhcp4: yes
+            dhcp6: yes
+            dhcp-identifier: mac
+        ens224:
+            dhcp4: no
+            dhcp6: no
+            addresses: [${private_ip_address}]
+EOS
+  netplan apply
+}
+configure_private_ip_address
+
+#################################################
+##
+## SSH
+##
+configure_ssh_port () {
+  sed -i 's/^#*Port [0-9]*/Port ${ssh_port}/' /etc/ssh/sshd_config
+}
+configure_ssh_port
+
+#################################################
+##
+## Hostname
+##
+hostnamectl set-hostname ${hostname}
+
+#################################################
+##
+## Disable swap files genereated by systemd-gpt-auto-generator
+##
+systemctl mask "dev-sda3.swap"
@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">=1.3.7"
+  required_providers {
+    nifcloud = {
+      source  = "nifcloud/nifcloud"
+      version = ">= 1.8.0, < 2.0.0"
+    }
+  }
+}
@@ -0,0 +1,81 @@
+variable "availability_zone" {
+  description = "The availability zone"
+  type        = string
+}
+
+variable "prefix" {
+  description = "The prefix for the entire cluster"
+  type        = string
+  validation {
+    condition     = length(var.prefix) <= 5
+    error_message = "Must be a less than 5 character long."
+  }
+}
+
+variable "private_network_cidr" {
+  description = "The subnet of private network"
+  type        = string
+  validation {
+    condition     = can(cidrnetmask(var.private_network_cidr))
+    error_message = "Must be a valid IPv4 CIDR block address."
+  }
+}
+
+variable "private_ip_bn" {
+  description = "Private IP of bastion server"
+  type        = string
+}
+
+variable "instances_cp" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instances_wk" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instance_key_name" {
+  description = "The key name of the Key Pair to use for the instance"
+  type        = string
+}
+
+variable "instance_type_bn" {
+  description = "The instance type of bastion server"
+  type        = string
+}
+
+variable "instance_type_wk" {
+  description = "The instance type of worker"
+  type        = string
+}
+
+variable "instance_type_cp" {
+  description = "The instance type of control plane"
+  type        = string
+}
+
+variable "image_name" {
+  description = "The name of image"
+  type        = string
+}
+
+variable "additional_lb_filter" {
+  description = "Additional LB filter"
+  type        = list(string)
+}
+
+variable "accounting_type" {
+  type    = string
+  default = "1"
+  validation {
+    condition = anytrue([
+      var.accounting_type == "1", // Monthly
+      var.accounting_type == "2", // Pay per use
+    ])
+    error_message = "Must be a 1 or 2."
+  }
+}
@@ -0,0 +1,3 @@
+output "kubernetes_cluster" {
+  value = module.kubernetes_cluster
+}
@@ -0,0 +1,22 @@
+region = "jp-west-1"
+az     = "west-11"
+
+instance_key_name = "deployerkey"
+
+instance_type_bn = "e-medium"
+instance_type_cp = "e-medium"
+instance_type_wk = "e-medium"
+
+private_network_cidr = "192.168.30.0/24"
+instances_cp = {
+  "cp01" : { private_ip : "192.168.30.11/24" }
+  "cp02" : { private_ip : "192.168.30.12/24" }
+  "cp03" : { private_ip : "192.168.30.13/24" }
+}
+instances_wk = {
+  "wk01" : { private_ip : "192.168.30.21/24" }
+  "wk02" : { private_ip : "192.168.30.22/24" }
+}
+private_ip_bn = "192.168.30.10/24"
+
+image_name = "Ubuntu Server 22.04 LTS"
@@ -0,0 +1 @@
+../../../../inventory/sample/group_vars
@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">=1.3.7"
+  required_providers {
+    nifcloud = {
+      source  = "nifcloud/nifcloud"
+      version = "1.8.0"
+    }
+  }
+}
@@ -0,0 +1,77 @@
+variable "region" {
+  description = "The region"
+  type        = string
+}
+
+variable "az" {
+  description = "The availability zone"
+  type        = string
+}
+
+variable "private_ip_bn" {
+  description = "Private IP of bastion server"
+  type        = string
+}
+
+variable "private_network_cidr" {
+  description = "The subnet of private network"
+  type        = string
+  validation {
+    condition     = can(cidrnetmask(var.private_network_cidr))
+    error_message = "Must be a valid IPv4 CIDR block address."
+  }
+}
+
+variable "instances_cp" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instances_wk" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instance_key_name" {
+  description = "The key name of the Key Pair to use for the instance"
+  type        = string
+}
+
+variable "instance_type_bn" {
+  description = "The instance type of bastion server"
+  type        = string
+}
+
+variable "instance_type_wk" {
+  description = "The instance type of worker"
+  type        = string
+}
+
+variable "instance_type_cp" {
+  description = "The instance type of control plane"
+  type        = string
+}
+
+variable "image_name" {
+  description = "The name of image"
+  type        = string
+}
+
+variable "working_instance_ip" {
+  description = "The IP address to connect to bastion server."
+  type        = string
+}
+
+variable "accounting_type" {
+  type    = string
+  default = "2"
+  validation {
+    condition = anytrue([
+      var.accounting_type == "1", // Monthly
+      var.accounting_type == "2", // Pay per use
+    ])
+    error_message = "Must be a 1 or 2."
+  }
+}
@@ -8,11 +8,6 @@ OpenStack.
 This will install a Kubernetes cluster on an OpenStack Cloud. It should work on
 most modern installs of OpenStack that support the basic services.

-## OpenStack Support Documentation
-
-For OpenStack cloud provider configuration (Octavia load balancers, Cinder CSI storage,
-network plugins like Calico), see the [main OpenStack documentation](../../../docs/cloud_controllers/openstack.md).
-
 ### Known compatible public clouds

 - [Auro](https://auro.io/)
@@ -94,8 +89,8 @@ binaries available on hyperkube v1.4.3_coreos.0 or higher.

 ## Requirements

- [Install OpenTofu](https://opentofu.org/docs/intro/install/) 1.9.0 or later
- [Install Ansible](https://docs.ansible.com/ansible/latest/intro_installation.html)
+- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.14 or later
+- [Install Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html)
 - you already have a suitable OS image in Glance
 - you already have a floating IP pool created
 - you have security groups enabled
@@ -134,7 +129,6 @@ Terraform will be used to provision all of the OpenStack resources with base sof
 Create an inventory directory for your cluster by copying the existing sample and linking the `hosts` script (used to build the inventory based on Terraform state):

 ```ShellSession
-CLUSTER=your-cluster-name
 cp -LRp contrib/terraform/openstack/sample-inventory inventory/$CLUSTER
 cd inventory/$CLUSTER
 ln -s ../../contrib/terraform/openstack/hosts
@@ -152,7 +146,7 @@ different OpenStack environments may support Identity API version 2 or 3.

 These are examples and may vary depending on your OpenStack cloud provider,
 for an exhaustive list on how to authenticate on OpenStack with Terraform
-please read the [OpenStack provider documentation](https://registry.terraform.io/providers/terraform-provider-openstack/openstack/latest/docs).
+please read the [OpenStack provider documentation](https://www.terraform.io/docs/providers/openstack/).

 ##### Declarative method (recommended)

@@ -268,7 +262,6 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`image_uuid`,`image_gfs_uuid`, `image_master_uuid` | UUID of the image to use in provisioning the compute resources. Should already be loaded into glance. |
 |`ssh_user`,`ssh_user_gfs` | The username to ssh into the image with. This usually depends on the image you have selected |
 |`public_key_path` | Path on your local workstation to the public key file you wish to use in creating the key pairs |
-|`group_vars_path` | path to the inventory group vars directory, `./group_vars` by default |
 |`number_of_k8s_masters`, `number_of_k8s_masters_no_floating_ip` | Number of nodes that serve as both master and etcd. These can be provisioned with or without floating IP addresses|
 |`number_of_k8s_masters_no_etcd`, `number_of_k8s_masters_no_floating_ip_no_etcd` |  Number of nodes that serve as just master with no etcd. These can be provisioned with or without floating IP addresses |
 |`number_of_etcd` | Number of pure etcd nodes |
@@ -288,9 +281,9 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`k8s_allowed_remote_ips_ipv6` | List of IPv6 CIDR allowed to initiate a SSH connection, empty by default |
 |`k8s_allowed_egress_ipv6_ips` | List of IPv6 CIDRs allowed for egress traffic, `["::/0"]` by default |
 |`worker_allowed_ports` | List of ports to open on worker nodes, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "0.0.0.0/0"}]` by default |
-|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}, { "protocol" = "ipv6-icmp", "port_range_min" = 0, "port_range_max" = 0, "remote_ip_prefix" = "::/0"}]` by default |
+|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}]` by default |
 |`master_allowed_ports` | List of ports to open on master nodes, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "0.0.0.0/0"}]`, empty by default |
-|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, `[{ "protocol" = "ipv6-icmp", "port_range_min" = 0, "port_range_max" = 0, "remote_ip_prefix" = "::/0"}]` by default |
+|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "::/0"}]`, empty by default |
 |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |
 |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |
 |`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |
@@ -618,13 +611,7 @@ Edit `inventory/$CLUSTER/group_vars/all/all.yml`:
 bin_dir: /opt/bin
 ```

- **external_cloud_provider**:
-
-```yml
-external_cloud_provider: openstack
-```
-
- **Only if K8s < v1.31 - cloud_provider**:
+- and **cloud_provider**:

 ```yml
 cloud_provider: openstack
@@ -730,12 +717,6 @@ Basically you will install Gluster as
 ansible-playbook --become -i inventory/$CLUSTER/hosts ./contrib/network-storage/glusterfs/glusterfs.yml
 ```

-## Relevant Resources
-
- [HauptJ - Example cluster.tfvars using floating IPs for all Master and Nodes](https://gist.github.com/HauptJ/d72e2a8fe0698d448283a51e847a5dfa)
- [openmetal - Deploying a Kubespray cluster to OpenStack using Terraform](https://openmetal.io/docs/manuals/kubernetes-guides/deploying-a-kubespray-cluster-to-openstack-using-terraform/)
- [Guoqiang Lan - Deploy Kubernetes with Kubespray on OpenStack](https://guoqianglan.github.io/tutorial/cloud/deploy-kubernetes-with-kubespray-on-openstack/)
-
 ## What's next

 Try out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/).
@@ -1006,7 +1006,7 @@ resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" {
  name              = "${var.cluster_name}-gfs-node-nf-${count.index + 1}"
  count             = var.number_of_gfs_nodes_no_floating_ip
  availability_zone = element(var.az_list, count.index)
-  image_id          = var.gfs_root_volume_size_in_gb == 0 ? local.image_to_use_gfs : null
+  image_name        = var.gfs_root_volume_size_in_gb == 0 ? local.image_to_use_gfs : null
  flavor_id         = var.flavor_gfs_node
  key_pair          = openstack_compute_keypair_v2.k8s.name

@@ -1078,7 +1078,7 @@ resource "openstack_networking_floatingip_associate_v2" "k8s_nodes" {
  port_id               = openstack_networking_port_v2.k8s_nodes_port[each.key].id
 }

-resource "openstack_blockstorage_volume_v3" "glusterfs_volume" {
+resource "openstack_blockstorage_volume_v2" "glusterfs_volume" {
  name        = "${var.cluster_name}-glusterfs_volume-${count.index + 1}"
  count       = var.gfs_root_volume_size_in_gb == 0 ? var.number_of_gfs_nodes_no_floating_ip : 0
  description = "Non-ephemeral volume for GlusterFS"
@@ -1088,5 +1088,5 @@ resource "openstack_blockstorage_volume_v3" "glusterfs_volume" {
 resource "openstack_compute_volume_attach_v2" "glusterfs_volume" {
  count       = var.gfs_root_volume_size_in_gb == 0 ? var.number_of_gfs_nodes_no_floating_ip : 0
  instance_id = element(openstack_compute_instance_v2.glusterfs_node_no_floating_ip.*.id, count.index)
-  volume_id   = element(openstack_blockstorage_volume_v3.glusterfs_volume.*.id, count.index)
+  volume_id   = element(openstack_blockstorage_volume_v2.glusterfs_volume.*.id, count.index)
 }
@@ -271,14 +271,7 @@ variable "master_allowed_ports" {
 variable "master_allowed_ports_ipv6" {
  type = list(any)

-  default = [
-    {
-      "protocol"         = "ipv6-icmp"
-      "port_range_min"   = 0
-      "port_range_max"   = 0
-      "remote_ip_prefix" = "::/0"
-    },
-  ]
+  default = []
 }

 variable "worker_allowed_ports" {
@@ -304,12 +297,6 @@ variable "worker_allowed_ports_ipv6" {
      "port_range_max"   = 32767
      "remote_ip_prefix" = "::/0"
    },
-    {
-      "protocol"         = "ipv6-icmp"
-      "port_range_min"   = 0
-      "port_range_max"   = 0
-      "remote_ip_prefix" = "::/0"
-    },
  ]
 }

@@ -3,7 +3,7 @@ terraform {
  required_providers {
    upcloud = {
      source  = "UpCloudLtd/upcloud"
-      version = "~>5.29.1"
+      version = "~>5.9.0"
    }
  }
  required_version = ">= 0.13"
@@ -3,7 +3,7 @@ terraform {
  required_providers {
    upcloud = {
      source  = "UpCloudLtd/upcloud"
-      version = "~>5.29.1"
+      version = "~>5.9.0"
    }
  }
  required_version = ">= 0.13"
@@ -1,13 +1,5 @@
 # Cilium

-## Unprivileged agent configuration
-
-By default, Cilium is installed with `securityContext.privileged: false`. You need to set the `kube_owner` variable to `root` in the inventory:
-
-```yml
-kube_owner: root
-```
-
 ## IP Address Management (IPAM)

 IP Address Management (IPAM) is responsible for the allocation and management of IP addresses used by network endpoints (container and others) managed by Cilium. The default mode is "Cluster Scope".
@@ -245,7 +237,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version

 ```yml
-cilium_version: "1.19.4"
+cilium_version: "1.18.4"
 ```

 ## Add variable to config
@@ -32,7 +32,7 @@ add `kube_proxy_masquerade_all: true` in `group_vars/all/all.yml`

 * Disable nodelocaldns

-The nodelocal dns IP is not reachable.
+The nodelocal dns IP is not reacheable.

 Disable it in `sample/group_vars/k8s_cluster/k8s_cluster.yml`

@@ -65,8 +65,9 @@ In kubespray, the default runtime name is "runc", and it can be configured with
 containerd_runc_runtime:
  name: runc
  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
  options:
-    Root: ""
    SystemdCgroup: "false"
    BinaryName: /usr/local/bin/my-runc
  base_runtime_spec: cri-base.json
@@ -80,7 +80,7 @@ The `crio_remap_enable` configures the `/etc/subuid` and `/etc/subgid` files to
 By default, 16M uids and gids are reserved for user namespaces (256 pods * 65536 uids/gids) at the end of the uid/gid space.

 The `crio_default_capabilities` configure the default containers capabilities for the crio.
-Defaults capabilities are:
+Defaults capabilties are:

 ```yaml
 crio_default_capabilities:
@@ -6,6 +6,7 @@
  * [Downloads](/docs/advanced/downloads.md)
  * [Gcp-lb](/docs/advanced/gcp-lb.md)
  * [Kubernetes-reliability](/docs/advanced/kubernetes-reliability.md)
+  * [Mitogen](/docs/advanced/mitogen.md)
  * [Netcheck](/docs/advanced/netcheck.md)
  * [Ntp](/docs/advanced/ntp.md)
  * [Proxy](/docs/advanced/proxy.md)
@@ -57,6 +58,7 @@
  * [Setting-up-your-first-cluster](/docs/getting_started/setting-up-your-first-cluster.md)
 * Ingress
  * [Alb Ingress Controller](/docs/ingress/alb_ingress_controller.md)
+  * [Ingress Nginx](/docs/ingress/ingress_nginx.md)
  * [Kube-vip](/docs/ingress/kube-vip.md)
  * [Metallb](/docs/ingress/metallb.md)
 * Operating Systems
@@ -6,7 +6,7 @@
    - [Create New TLS Root CA Certificate and Key](#create-new-tls-root-ca-certificate-and-key)
      - [Install Cloudflare PKI/TLS `cfssl` Toolkit.](#install-cloudflare-pkitls-cfssl-toolkit)
      - [Create Root Certificate Authority (CA) Configuration File](#create-root-certificate-authority-ca-configuration-file)
-      - [Create Certificate Signing Request (CSR) Configuration File](#create-certificate-signing-request-csr-configuration-file)
+      - [Create Certficate Signing Request (CSR) Configuration File](#create-certficate-signing-request-csr-configuration-file)
      - [Create TLS Root CA Certificate and Key](#create-tls-root-ca-certificate-and-key)

 Cert-Manager is a native Kubernetes certificate management controller. It can help with issuing certificates from a variety of sources, such as Let’s Encrypt, HashiCorp Vault, Venafi, a simple signing key pair, or self signed. It will ensure certificates are valid and up to date, and attempt to renew certificates at a configured time before expiry.
@@ -30,7 +30,14 @@ If you don't have a TLS Root CA certificate and key available, you can create th

 A common use-case for cert-manager is requesting TLS signed certificates to secure your ingress resources. This can be done by simply adding annotations to your Ingress resources and cert-manager will facilitate creating the Certificate resource for you. A small sub-component of cert-manager, ingress-shim, is responsible for this.

-For example, if you're using the Traefik ingress controller, you can secure the Prometheus ingress by adding the annotation `cert-manager.io/cluster-issuer: ca-issuer` and the `spec.tls` section to the `Ingress` resource definition.
+To enable the Nginx Ingress controller as part of your Kubespray deployment, simply edit your K8s cluster addons inventory e.g. `inventory\sample\group_vars\k8s_cluster\addons.yml` and set `ingress_nginx_enabled` to true.
+
+```ini
+# Nginx ingress controller deployment
+ingress_nginx_enabled: true
+```
+
+For example, if you're using the Nginx ingress controller, you can secure the Prometheus ingress by adding the annotation `cert-manager.io/cluster-issuer: ca-issuer` and the `spec.tls` section to the `Ingress` resource definition.

 ```yaml
 apiVersion: networking.k8s.io/v1
@@ -41,9 +48,9 @@ metadata:
  labels:
    prometheus: k8s
  annotations:
+    kubernetes.io/ingress.class: "nginx"
    cert-manager.io/cluster-issuer: ca-issuer
 spec:
-  ingressClassName: "traefik"
  tls:
  - hosts:
    - prometheus.example.com
@@ -65,8 +72,8 @@ Once deployed to your K8s cluster, every 3 months cert-manager will automaticall

 Please consult the official upstream documentation:

- [cert-manager Ingress Usage](https://cert-manager.io/usage/ingress/)
- [cert-manager Ingress Tutorial](https://cert-manager.io/tutorials/acme/ingress/#step-3-assign-a-dns-name)
+- [cert-manager Ingress Usage](https://cert-manager.io/v1.5-docs/usage/ingress/)
+- [cert-manager Ingress Tutorial](https://cert-manager.io/v1.5-docs/tutorials/acme/ingress/#step-3-assign-a-dns-name)

 ### ACME

@@ -74,12 +81,12 @@ The ACME Issuer type represents a single account registered with the Automated C

 Certificates issued by public ACME servers are typically trusted by client’s computers by default. This means that, for example, visiting a website that is backed by an ACME certificate issued for that URL, will be trusted by default by most client’s web browsers. ACME certificates are typically free.

- [ACME Configuration](https://cert-manager.io/docs/configuration/acme/)
- [ACME HTTP Validation](https://cert-manager.io/docs/tutorials/acme/http-validation/)
-  - [HTTP01 Challenges](https://cert-manager.io/docs/configuration/acme/http01/)
- [ACME DNS Validation](https://cert-manager.io/docs/tutorials/acme/dns-validation/)
-  - [DNS01 Challenges](https://cert-manager.io/docs/configuration/acme/dns01/)
- [ACME FAQ](https://cert-manager.io/docs/troubleshooting/acme/)
+- [ACME Configuration](https://cert-manager.io/v1.5-docs/configuration/acme/)
+- [ACME HTTP Validation](https://cert-manager.io/v1.5-docs/tutorials/acme/http-validation/)
+  - [HTTP01 Challenges](https://cert-manager.io/v1.5-docs/configuration/acme/http01/)
+- [ACME DNS Validation](https://cert-manager.io/v1.5-docs/tutorials/acme/dns-validation/)
+  - [DNS01 Challenges](https://cert-manager.io/v1.5-docs/configuration/acme/dns01/)
+- [ACME FAQ](https://cert-manager.io/v1.5-docs/faq/acme/)

 #### ACME With An Internal Certificate Authority

@@ -127,7 +134,7 @@ $ cat > ca-config.json <<EOF
 EOF
 ```

-#### Create Certificate Signing Request (CSR) Configuration File
+#### Create Certficate Signing Request (CSR) Configuration File

 The TLS certificate `names` details can be updated to your own specific requirements.

@@ -1,4 +1,4 @@
-# GCP Load Balancers for type=LoadBalancer of Kubernetes Services
+# GCP Load Balancers for type=LoadBalacer of Kubernetes Services

 > **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider)

@@ -0,0 +1,30 @@
+# Mitogen
+
+*Warning:* Mitogen support is now deprecated in kubespray due to upstream not releasing an updated version to support ansible 4.x (ansible-base 2.11.x) and above. The CI support has been stripped for mitogen and we are no longer validating any support or regressions for it. The supporting mitogen install playbook and integration documentation will be removed in a later version.
+
+[Mitogen for Ansible](https://mitogen.networkgenomics.com/ansible_detailed.html) allow a 1.25x - 7x speedup and a CPU usage reduction of at least 2x, depending on network conditions, modules executed, and time already spent by targets on useful work. Mitogen cannot improve a module once it is executing, it can only ensure the module executes as quickly as possible.
+
+## Install
+
+```ShellSession
+ansible-playbook contrib/mitogen/mitogen.yml
+```
+
+The above playbook sets the ansible `strategy` and `strategy_plugins` in `ansible.cfg` but you can also enable them if you use your own `ansible.cfg` by setting the environment varialbles:
+
+```ShellSession
+export ANSIBLE_STRATEGY=mitogen_linear
+export ANSIBLE_STRATEGY_PLUGINS=plugins/mitogen/ansible_mitogen/plugins/strategy
+```
+
+... or `ansible.cfg` setup:
+
+```ini
+[defaults]
+strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy
+strategy=mitogen_linear
+```
+
+## Limitation
+
+If you are experiencing problems, please see the [documentation](https://mitogen.networkgenomics.com/ansible_detailed.html#noteworthy-differences).
@@ -30,9 +30,9 @@ If the latest version supported according to pip is 6.7.0 it means you are runni

 Based on the table below and the available python version for your ansible host you should choose the appropriate ansible version to use with kubespray.

-|  Ansible Version  | Python Version |
-|-------------------|----------------|
-| >=2.18.0, <2.19.0 | 3.11-3.13      |
+| Ansible Version | Python Version |
+|-----------------|----------------|
+| >= 2.17.3       | 3.10-3.12      |

 ## Customize Ansible vars

@@ -42,10 +42,13 @@ Kubespray expects users to use one of the following variables sources for settin
 |----------------------------------------|------------------------------------------------------------------------------|
 | inventory vars                         |                                                                              |
 |  - **inventory group_vars**            | most used                                                                    |
-|  - inventory host_vars                 | host specific vars overrides, group_vars is usually more practical           |
+|  - inventory host_vars                 | host specifc vars overrides, group_vars is usually more practical            |
 | **extra vars** (always win precedence) | override with ``ansible-playbook -e @foo.yml``                               |

-> Extra vars are best used to override kubespray internal variables, for instances, roles/vars/. Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray interface. Thus they can change, disappear, or break stuff unexpectedly.
+[!IMPORTANT]
+Extra vars are best used to override kubespray internal variables, for instances, roles/vars/.
+Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray
+interface. Thus they can change, disappear, or break stuff unexpectedly.

 ## Ansible tags

@@ -78,6 +81,7 @@ The following tags are defined in playbooks:
 | crio                           | Configuring crio container engine for hosts           |
 | crun                           | Configuring crun runtime                              |
 | csi-driver                     | Configuring csi driver                                |
+| dashboard                      | Installing and configuring the Kubernetes Dashboard   |
 | dns                            | Remove dns entries when resetting                     |
 | docker                         | Configuring docker engine runtime for hosts           |
 | download                       | Fetching container images to a delegate host          |
@@ -118,7 +122,7 @@ The following tags are defined in playbooks:
 | metrics_server                 | Configuring metrics_server                            |
 | netchecker                     | Installing netchecker K8s app                         |
 | network                        | Configuring networking plugins for K8s                |
-| mounts                         | Umount kubelet dirs when resetting                    |
+| mounts                         | Umount kubelet dirs when reseting                     |
 | multus                         | Network plugin multus                                 |
 | nginx                          | Configuring LB for kube-apiserver instances           |
 | node                           | Configuring K8s minion (compute) node role            |
@@ -157,33 +161,37 @@ Example command to filter and apply only DNS configuration tasks and skip
 everything else related to host OS configuration and downloading images of containers:

 ```ShellSession
-ansible-playbook -i inventory/sample/inventory.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os
+ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os
 ```

 And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files:

 ```ShellSession
-ansible-playbook -i inventory/sample/inventory.ini -e dns_mode='none' cluster.yml --tags resolvconf
+ansible-playbook -i inventory/sample/hosts.ini -e dns_mode='none' cluster.yml --tags resolvconf
 ```

 And this prepares all container images locally (at the ansible runner node) without installing
 or upgrading related stuff or trying to upload container to K8s cluster nodes:

 ```ShellSession
-ansible-playbook -i inventory/sample/inventory.ini cluster.yml \
+ansible-playbook -i inventory/sample/hosts.ini cluster.yml \
    -e download_run_once=true -e download_localhost=true \
    --tags download --skip-tags upload,upgrade
 ```

 Note: use `--tags` and `--skip-tags` wisely and only if you're 100% sure what you're doing.

+## Mitogen
+
+Mitogen support is deprecated, please see [mitogen related docs](/docs/advanced/mitogen.md) for usage and reasons for deprecation.
+
 ## Troubleshooting Ansible issues

 Having the wrong version of ansible, ansible collections or python dependencies can cause issue.
-In particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRARY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection)
+In particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRAY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection)

 ```ShellSession
-export ANSIBLE_LIBRARY=<kubespray_dir>/library`
+export ANSIBLE_LIBRAY=<kubespray_dir>/library`
 ```

 A simple way to ensure you get all the correct version of Ansible is to use
@@ -192,11 +200,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:

 ```ShellSession
-git checkout v2.30.0
-docker pull quay.io/kubespray/kubespray:v2.30.0
+git checkout v2.29.0
+docker pull quay.io/kubespray/kubespray:v2.29.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.30.0 bash
+  quay.io/kubespray/kubespray:v2.29.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -62,7 +62,7 @@ Once the configuration is set, you can execute the playbook again to apply the n

 ```ShellSession
 cd kubespray
-ansible-playbook -i inventory/sample/inventory.ini -b -v cluster.yml
+ansible-playbook -i inventory/sample/hosts.ini -b -v cluster.yml
 ```

 You'll find some useful examples [here](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#sample-manifests-to-test-csi-driver-functionality) to test your configuration.
@@ -82,7 +82,7 @@ If you intend to leverage the [zone and region node labeling](https://kubernetes

 ### Kubespray configuration (deprecated)

-First you must define the cloud provider in `inventory/sample/group_vars/all/all.yml` and set it to `vsphere`.
+First you must define the cloud provider in `inventory/sample/group_vars/all.yml` and set it to `vsphere`.

 ```yml
 cloud_provider: vsphere
@@ -128,7 +128,7 @@ Once the configuration is set, you can execute the playbook again to apply the n

 ```ShellSession
 cd kubespray
-ansible-playbook -i inventory/sample/inventory.ini -b -v cluster.yml
+ansible-playbook -i inventory/sample/hosts.ini -b -v cluster.yml
 ```

 You'll find some useful examples [here](https://github.com/kubernetes/examples/tree/master/staging/volumes/vsphere) to test your configuration.
@@ -11,5 +11,5 @@ You can deploy instances in your cloud environment in several ways. Examples inc
 With ansible-playbook command

 ```ShellSession
-ansible-playbook -u smana -e ansible_ssh_user=admin -e cloud_provider=[aws|gce] -b --become-user=root -i inventory/sample/inventory.ini cluster.yml
+ansible-playbook -u smana -e ansible_ssh_user=admin -e cloud_provider=[aws|gce] -b --become-user=root -i inventory/single.cfg cluster.yml
 ```
@@ -6,7 +6,7 @@ See [.gitlab-ci.yml](/.gitlab-ci.yml) and the included files for an overview.

 ## Runners

-Kubespray has 2 types of GitLab runners, both deployed on the Kubespray CI cluster (hosted on Oracle Cloud Infrastructure):
+Kubespray has 2 types of GitLab runners, both deployed  on the Kubespray CI cluster (hosted on Oracle Cloud Infrastucture):

 - pods: use the [gitlab-ci kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes/)
 - vagrant: custom executor running in pods with access to the libvirt socket on the nodes
@@ -145,6 +145,7 @@ upstream_dns_servers:
  - 1.0.0.1

 # Extensions
+ingress_nginx_enabled: True
 helm_enabled: True
 cert_manager_enabled: True
 metrics_server_enabled: True
@@ -155,7 +156,7 @@ kube_feature_gates:
  - "NodeSwap=True"
 ```

-## Additional files
+## Aditional files

 This section documents additional files used to complete a deployment of the kubespray CI, these files sit on the control-plane node and assume a working kubernetes cluster.

@@ -11,12 +11,12 @@ amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
 debian13 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-fedora42 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
-fedora43 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |

@@ -29,14 +29,14 @@ amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora42 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora43 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |

 ## docker

@@ -47,11 +47,11 @@ amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora42 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora43 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -15,15 +15,16 @@ and [details on the inventory structure expected by Kubespray](/docs/ansible/inv
 <your-favorite-editor> inventory/mycluster/inventory.ini

 # Review and change parameters under ``inventory/mycluster/group_vars``
-<your-favorite-editor> inventory/mycluster/group_vars/all/all.yml # for every node, including etcd
-<your-favorite-editor> inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml # for every node in the cluster (not etcd when it's separate)
-<your-favorite-editor> inventory/mycluster/group_vars/k8s_cluster/kube_control_plane.yml # for the control plane
+<your-favorite-editor> inventory/mycluster/group_vars/all.yml # for every node, including etcd
+<your-favorite-editor> inventory/mycluster/group_vars/k8s_cluster.yml # for every node in the cluster (not etcd when it's separate)
+<your-favorite-editor> inventory/mycluster/group_vars/kube_control_plane.yml # for the control plane
+<your-favorite-editor> inventory/myclsuter/group_vars/kube_node.yml # for worker nodes
 ```

 ## Installing the cluster

 ```ShellSession
-ansible-playbook -i inventory/mycluster/inventory.ini cluster.yml -b -v \
+ansible-playbook -i inventory/mycluster/ cluster.yml -b -v \
  --private-key=~/.ssh/private_key
 ```

@@ -35,7 +36,7 @@ You may want to add worker, control plane or etcd nodes to your existing cluster
 - Run the ansible-playbook command, substituting `cluster.yml` for `scale.yml`:

 ```ShellSession
-ansible-playbook -i inventory/mycluster/inventory.ini scale.yml -b -v \
+ansible-playbook -i inventory/mycluster/hosts.yml scale.yml -b -v \
  --private-key=~/.ssh/private_key
 ```

@@ -53,7 +54,7 @@ is not working, you can remove the node and install it again.
 Use `--extra-vars "node=<nodename>,<nodename2>"` to select the node(s) you want to delete.

 ```ShellSession
-ansible-playbook -i inventory/mycluster/inventory.ini remove-node.yml -b -v \
+ansible-playbook -i inventory/mycluster/hosts.yml remove-node.yml -b -v \
 --private-key=~/.ssh/private_key \
 --extra-vars "node=nodename,nodename2"
 ```
@@ -82,11 +83,37 @@ authentication. One can get a kubeconfig from kube_control_plane hosts
 For more information on kubeconfig and accessing a Kubernetes cluster, refer to
 the Kubernetes [documentation](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/).

+## Accessing Kubernetes Dashboard
+
+Supported version is kubernetes-dashboard v2.0.x :
+
+- Login option : token/kubeconfig by default
+- Deployed by default in "kube-system" namespace, can be overridden with `dashboard_namespace: kubernetes-dashboard` in inventory,
+- Only serves over https
+
+Access is described in [dashboard docs](https://github.com/kubernetes/dashboard/tree/master/docs/user/accessing-dashboard). With kubespray's default deployment in kube-system namespace, instead of kubernetes-dashboard :
+
+- Proxy URL is <http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#/login>
+- kubectl commands must be run with "-n kube-system"
+
+Accessing through Ingress is highly recommended. For proxy access, please note that proxy must listen to [localhost](https://github.com/kubernetes/dashboard/issues/692#issuecomment-220492484) (`proxy  --address="x.x.x.x"` will not work)
+
+For token authentication, guide to create Service Account is provided in [dashboard sample user](https://github.com/kubernetes/dashboard/blob/master/docs/user/access-control/creating-sample-user.md) doc. Still take care of default namespace.
+
+Access can also by achieved via ssh tunnel on a control plane :
+
+```bash
+# localhost:8081 will be sent to control-plane-1's own localhost:8081
+ssh -L8001:localhost:8001 user@control-plane-1
+sudo -i
+kubectl proxy
+```
+
 ## Accessing Kubernetes API

 The main client of Kubernetes is `kubectl`. It is installed on each kube_control_plane
 host and can optionally be configured on your ansible host by setting
-`kubectl_localhost: true` and `kubeconfig_localhost: true` in `inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml`:
+`kubectl_localhost: true` and `kubeconfig_localhost: true` in the configuration:

 - If `kubectl_localhost` enabled, `kubectl` will download onto `/usr/local/bin/` and setup with bash completion. A helper script `inventory/mycluster/artifacts/kubectl.sh` also created for setup with below `admin.conf`.
 - If `kubeconfig_localhost` enabled `admin.conf` will appear in the `inventory/mycluster/artifacts/` directory after deployment.
@@ -223,7 +223,7 @@ that controller-0, controller-1 and controller-2 in the `kube_control_plane` gro
 worker-0, worker-1 and worker-2 in the `kube_node` group. Add respective `ip` to the respective local VPC IP for each node.

 The main configuration for the cluster is stored in
-`inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml`. In this file we
+`inventory/mycluster/group_vars/k8s_cluster/k8s_cluster.yml`. In this file we
 will update the `supplementary_addresses_in_ssl_keys` with a list of the IP
 addresses of the controller nodes. In that way we can access the
  kubernetes API server as an administrator from outside the VPC network. You
@@ -240,7 +240,7 @@ the kubernetes cluster, just change the 'false' to 'true' for
 Now we will deploy the configuration:

 ```ShellSession
-ansible-playbook -i inventory/mycluster/inventory.ini -u $USERNAME -b -v --private-key=~/.ssh/id_rsa cluster.yml
+ansible-playbook -i inventory/mycluster/ -u $USERNAME -b -v --private-key=~/.ssh/id_rsa cluster.yml
 ```

 Ansible will now execute the playbook, this can take up to 20 minutes.
@@ -594,7 +594,7 @@ If you want to keep the VMs and just remove the cluster state, you can simply
 run another Ansible playbook:

 ```ShellSession
-ansible-playbook -i inventory/mycluster/inventory.ini -u $USERNAME -b -v --private-key=~/.ssh/id_rsa reset.yml
+ansible-playbook -i inventory/mycluster/ -u $USERNAME -b -v --private-key=~/.ssh/id_rsa reset.yml
 ```

 Resetting the cluster to the VMs original state usually takes about a couple
@@ -0,0 +1,203 @@
+# Installation Guide
+
+## Contents
+
+- [Prerequisite Generic Deployment Command](#prerequisite-generic-deployment-command)
+  - [Provider Specific Steps](#provider-specific-steps)
+    - [Docker for Mac](#docker-for-mac)
+    - [minikube](#minikube)
+    - [AWS](#aws)
+    - [GCE - GKE](#gce-gke)
+    - [Azure](#azure)
+    - [Bare-metal](#bare-metal)
+  - [Verify installation](#verify-installation)
+  - [Detect installed version](#detect-installed-version)
+- [Using Helm](#using-helm)
+
+## Prerequisite Generic Deployment Command
+
+!!! attention
+    The default configuration watches Ingress object from *all the namespaces*.
+    To change this behavior use the flag `--watch-namespace` to limit the scope to a particular namespace.
+
+!!! warning
+    If multiple Ingresses define different paths for the same host, the ingress controller will merge the definitions.
+
+!!! attention
+    If you're using GKE you need to initialize your user as a cluster-admin with the following command:
+
+```console
+kubectl create clusterrolebinding cluster-admin-binding \
+--clusterrole cluster-admin \
+--user $(gcloud config get-value account)
+```
+
+The following **Mandatory Command** is required for all deployments except for AWS. See below for the AWS version.
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.13.3/deploy/static/provider/cloud/deploy.yaml
+```
+
+### Provider Specific Steps
+
+There are cloud provider specific yaml files.
+
+#### Docker for Mac
+
+Kubernetes is available in Docker for Mac (from [version 18.06.0-ce](https://docs.docker.com/docker-for-mac/release-notes/#stable-releases-of-2018))
+
+First you need to [enable kubernetes](https://docs.docker.com/docker-for-mac/#kubernetes).
+
+Then you have to create a service:
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/cloud-generic.yaml
+```
+
+#### minikube
+
+For standard usage:
+
+```console
+minikube addons enable ingress
+```
+
+For development:
+
+1. Disable the ingress addon:
+
+```console
+minikube addons disable ingress
+```
+
+1. Execute `make dev-env`
+1. Confirm the `nginx-ingress-controller` deployment exists:
+
+```console
+$ kubectl get pods -n ingress-nginx
+NAME                                       READY     STATUS    RESTARTS   AGE
+default-http-backend-66b447d9cf-rrlf9      1/1       Running   0          12s
+nginx-ingress-controller-fdcdcd6dd-vvpgs   1/1       Running   0          11s
+```
+
+#### AWS
+
+In AWS we use an Elastic Load Balancer (ELB) to expose the NGINX Ingress controller behind a Service of `Type=LoadBalancer`.
+Since Kubernetes v1.9.0 it is possible to use a classic load balancer (ELB) or network load balancer (NLB)
+Please check the [elastic load balancing AWS details page](https://aws.amazon.com/elasticloadbalancing/details/)
+
+##### Elastic Load Balancer - ELB
+
+This setup requires to choose in which layer (L4 or L7) we want to configure the Load Balancer:
+
+- [Layer 4](https://en.wikipedia.org/wiki/OSI_model#Layer_4:_Transport_Layer): Use an Network Load Balancer (NLB) with TCP as the listener protocol for ports 80 and 443.
+- [Layer 7](https://en.wikipedia.org/wiki/OSI_model#Layer_7:_Application_Layer): Use an Elastic Load Balancer (ELB) with HTTP as the listener protocol for port 80 and terminate TLS in the ELB
+
+For L4:
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/aws/deploy.yaml
+```
+
+For L7:
+
+Change the value of `service.beta.kubernetes.io/aws-load-balancer-ssl-cert` in the file `provider/aws/deploy-tls-termination.yaml` replacing the dummy id with a valid one. The dummy value is `"arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX"`
+
+Check that no change is necessary with regards to the ELB idle timeout. In some scenarios, users may want to modify the ELB idle timeout, so please check the [ELB Idle Timeouts section](#elb-idle-timeouts) for additional information. If a change is required, users will need to update the value of `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` in `provider/aws/deploy-tls-termination.yaml`
+
+Then execute:
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/aws/deploy-tls-termination.yaml
+```
+
+This example creates an ELB with just two listeners, one in port 80 and another in port 443
+
+![Listeners](https://github.com/kubernetes/ingress-nginx/blob/main/docs/images/elb-l7-listener.png)
+
+##### ELB Idle Timeouts
+
+In some scenarios users will need to modify the value of the ELB idle timeout.
+Users need to ensure the idle timeout is less than the [keepalive_timeout](http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) that is configured for NGINX.
+By default NGINX `keepalive_timeout` is set to `75s`.
+
+The default ELB idle timeout will work for most scenarios, unless the NGINX [keepalive_timeout](http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) has been modified,
+in which case `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` will need to be modified to ensure it is less than the `keepalive_timeout` the user has configured.
+
+*Please Note: An idle timeout of `3600s` is recommended when using WebSockets.*
+
+More information with regards to idle timeouts for your Load Balancer can be found in the [official AWS documentation](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/config-idle-timeout.html).
+
+##### Network Load Balancer (NLB)
+
+This type of load balancer is supported since v1.10.0 as an ALPHA feature.
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/aws/service-nlb.yaml
+```
+
+#### GCE-GKE
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/cloud-generic.yaml
+```
+
+**Important Note:** proxy protocol is not supported in GCE/GKE
+
+#### Azure
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/cloud-generic.yaml
+```
+
+#### Bare-metal
+
+Using [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport):
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/baremetal/deploy.yaml
+```
+
+!!! tip
+    For extended notes regarding deployments on bare-metal, see [Bare-metal considerations](https://github.com/kubernetes/ingress-nginx/blob/main/docs/deploy/baremetal.md).
+
+### Verify installation
+
+To check if the ingress controller pods have started, run the following command:
+
+```console
+kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx --watch
+```
+
+Once the operator pods are running, you can cancel the above command by typing `Ctrl+C`.
+Now, you are ready to create your first ingress.
+
+### Detect installed version
+
+To detect which version of the ingress controller is running, exec into the pod and run `nginx-ingress-controller version` command.
+
+```console
+POD_NAMESPACE=ingress-nginx
+POD_NAME=$(kubectl get pods -n $POD_NAMESPACE -l app.kubernetes.io/component=controller -o jsonpath='{.items[0].metadata.name}')
+
+kubectl exec -it $POD_NAME -n $POD_NAMESPACE -- /nginx-ingress-controller --version
+```
+
+## Using Helm
+
+NGINX Ingress controller can be installed via [Helm](https://helm.sh/) using the chart [ingress-nginx/ingress-nginx](https://kubernetes.github.io/ingress-nginx).
+Official documentation is [here](https://kubernetes.github.io/ingress-nginx/deploy/#using-helm)
+
+To install the chart with the release name `my-nginx`:
+
+```console
+helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+helm install my-nginx ingress-nginx/ingress-nginx
+```
+
+Detect installed version:
+
+```console
+POD_NAME=$(kubectl get pods -l app.kubernetes.io/name=ingress-nginx -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it $POD_NAME -- /nginx-ingress-controller --version
+```
@@ -63,8 +63,6 @@ kube_vip_bgppeers:
 # kube_vip_bgp_peeraddress:
 # kube_vip_bgp_peerpass:
 # kube_vip_bgp_peeras:
-# kube_vip_bgp_sourceip:
-# kube_vip_bgp_sourceif:
 ```

 If using [control plane load-balancing](https://kube-vip.io/docs/about/architecture/#control-plane-load-balancing):
@@ -86,12 +84,3 @@ kube_vip_leaseduration: 30
 kube_vip_renewdeadline: 20
 kube_vip_retryperiod: 4
 ```
-
-To expose [Prometheus metrics](https://kube-vip.io/docs/installation/flags/#environment-variables) from the kube-vip static pod, set `kube_vip_metrics_enabled`. `kube_vip_metrics_port` is an integer; the manifest sets `prometheus_server` to `:PORT` because kube-vip passes that value to Go's HTTP listen address (see [`servePrometheusHTTPServer` in kube-vip](https://github.com/kube-vip/kube-vip/blob/main/cmd/kube-vip.go)). The manifest `ports` entry uses the same number for tooling that reads the pod spec.
-
-Kubespray defaults `kube_vip_metrics_port` to `2112`, matching upstream kube-vip's `--prometheusHTTPServer` default. Override it if your scrape config expects another port.
-
-```yaml
-kube_vip_metrics_enabled: true
-# kube_vip_metrics_port: 2112
-```
@@ -21,12 +21,6 @@ metallb_enabled: true
 metallb_speaker_enabled: true
 ```

-By default, MetalLB resources are deployed into the `metallb-system` namespace. You can override this namespace using a variable.
-
-```yaml
-metallb_namespace: woodenlb-system
-```
-
 By default only the MetalLB BGP speaker is allowed to run on control plane nodes. If you have a single node cluster or a cluster where control plane are also worker nodes you may need to enable tolerations for the MetalLB controller:

 ```yaml
@@ -41,7 +35,7 @@ metallb_config:
      effect: "NoSchedule"
 ```

-If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fashion:
+If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fasion:

 ```yaml
 metallb_config:
@@ -6,9 +6,9 @@ The documentation also applies to Red Hat derivatives, including Alma Linux, Roc

 The content of this section does not apply to open-source derivatives.

-In order to install packages via yum or dnf, RHEL hosts are required to be registered for a valid Red Hat support subscription.
+In order to install packages via yum or dnf, RHEL 7/8 hosts are required to be registered for a valid Red Hat support subscription.

-You can apply for a 1-year Development support subscription by creating a [Red Hat Developers](https://developers.redhat.com/) account. Be aware though that as the Red Hat Developers subscription is limited to only 1 year, it should not be used to register RHEL hosts provisioned in Production environments.
+You can apply for a 1-year Development support subscription by creating a [Red Hat Developers](https://developers.redhat.com/) account. Be aware though that as the Red Hat Developers subscription is limited to only 1 year, it should not be used to register RHEL 7/8 hosts provisioned in Production environments.

 Once you have a Red Hat support account, simply add the credentials to the Ansible inventory parameters `rh_subscription_username` and `rh_subscription_password` prior to deploying Kubespray. If your company has a Corporate Red Hat support account, then obtain an **Organization ID** and **Activation Key**, and add these to the Ansible inventory parameters `rh_subscription_org_id` and `rh_subscription_activation_key` instead of using your Red Hat support account credentials.

@@ -29,12 +29,12 @@ rh_subscription_role: "Red Hat Enterprise Server"
 rh_subscription_sla: "Self-Support"
 ```

-If the RHEL hosts are already registered to a valid Red Hat support subscription via an alternative configuration management approach prior to the deployment of Kubespray, the successful RHEL `subscription-manager` status check will simply result in the RHEL subscription registration tasks being skipped.
+If the RHEL 8/9 hosts are already registered to a valid Red Hat support subscription via an alternative configuration management approach prior to the deployment of Kubespray, the successful RHEL `subscription-manager` status check will simply result in the RHEL subscription registration tasks being skipped.

-## Rocky Linux 10
+## RHEL 8

-(Experimental in Kubespray CI)
+If you have containers that are using iptables in the host network namespace (`hostNetwork=true`),
+you need to ensure they are using iptables-nft.
+An example how k8s do the autodetection can be found [in this PR](https://github.com/kubernetes/kubernetes/pull/82966)

-The official Rocky Linux 10 cloud image does not include `kernel-module-extra`. Both Kube Proxy and CNI rely on this package, and since it relates to kernel version compatibility (which may require VM reboots, etc.), we haven't found an ideal solution.
-
-However, some users report that it doesn't affect them (minimal version). Therefore, the Kubespray CI Rocky Linux 10 image is built by Kubespray maintainers using `diskimage-builder`. For detailed methods, please refer to [the comments](https://github.com/kubernetes-sigs/kubespray/pull/12355#issuecomment-3705400093).
+The kernel version is lower than the kubenretes 1.32 system validation, please refer to the [kernel requirements](../operations/kernel-requirements.md).
@@ -32,12 +32,12 @@ etcd_metrics_service_labels:
  k8s-app: etcd
  app.kubernetes.io/managed-by: Kubespray
  app: kube-prometheus-stack-kube-etcd
-  release: kube-prometheus-stack
+  release: prometheus-stack
 ```

 The last two labels in the above example allows to scrape the metrics from the
 [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack)
-chart when it is installed with the release name `kube-prometheus-stack` and the following Helm `values.yaml`:
+chart with the following Helm `values.yaml` :

 ```yaml
 kubeEtcd:
@@ -45,22 +45,8 @@ kubeEtcd:
    enabled: false
 ```

-If your Helm release name is different, adjust the `release` label accordingly.
-
-To fully override metrics exposition URLs, define it in the inventory with:
+To fully override metrics exposition urls, define it in the inventory with:

 ```yaml
 etcd_listen_metrics_urls: "http://0.0.0.0:2381"
 ```
-
-If you choose to expose metrics on specific node IPs (for example `10.141.4.22`, `10.141.4.23`, `10.141.4.24`) in `etcd_listen_metrics_urls`,
-you can configure kube-prometheus-stack to scrape those endpoints directly with:
-
-```yaml
-kubeEtcd:
-  enabled: true
-  endpoints:
-    - 10.141.4.22
-    - 10.141.4.23
-    - 10.141.4.24
-```
@@ -100,6 +100,8 @@ kubelet_make_iptables_util_chains: true
 kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
 kubelet_seccomp_default: true
 kubelet_systemd_hardening: true
+# To disable kubelet's staticPodPath (for nodes that don't use static pods like worker nodes)
+kubelet_static_pod_path: ""
 # In case you have multiple interfaces in your
 # control plane nodes and you want to specify the right
 # IP addresses, kubelet_secure_addresses allows you
@@ -11,7 +11,7 @@ kubeadm_ignore_preflight_errors:

 The Kernel Version Matrixs:

-| OS Version         | Kernel Version | Kernel >=4.19      |
+| OS Verion          | Kernel Verion  | Kernel >=4.19      |
 |---                 | ---            | ---                |
 | RHEL 9             | 5.14           | :white_check_mark: |
 | RHEL 8             | 4.18           | :x:                |
@@ -31,8 +31,6 @@ That's it.

 Append the new host to the inventory and run `cluster.yml`. You can NOT use `scale.yml` for that.

-**Note:** When adding new control plane nodes, always append them to the end of the `kube_control_plane` group in your inventory. Adding control plane nodes in the first position is not supported and will cause the playbook to fail.
-
 ### 2) Restart kube-system/nginx-proxy

 In all hosts, restart nginx-proxy pod. This pod is a local proxy for the apiserver. Kubespray will update its static config, but it needs to be restarted in order to reload.
@@ -22,10 +22,6 @@ Then you need to setup the following services on your offline environment:
 You can get artifact lists with [generate_list.sh](/contrib/offline/generate_list.sh) script.
 In addition, you can find some tools for offline deployment under [contrib/offline](/contrib/offline/README.md).

-## Tip: use the original domains as top directories in the files_repo, i.e `github.com/`, `dl.k8s.io/`, `storage.googleapis.com/`, `get.helm.sh/`
-
-## Tip: for Cilium ensure to mirror <https://helm.cilium.io/index.yaml> and the chart cilium-1.18.2.tgz in files_repo
-
 ## Access Control

 ### Note: access controlled files_repo
@@ -48,8 +44,7 @@ files_repo: "https://{{ files_repo_user ~ ':' ~ files_repo_pass ~ '@' ~ files_re

 ### Note: access controlled registry

-Specify a "{{ registry_user }}" and "{{ registry_pass }}" for "{{ registry_addr }}",
-These are used to download the container images. Ensure to encrypt the password (if used) with ansible-vault.
+To specify a username and password for "{{ registry_host }}", used to download the container images, you can use url-encoding too.

 ```yaml
 registry_pass: !vault |
@@ -59,30 +54,7 @@ registry_pass: !vault |
          64653965663965356137333436616536643132336630313235333232336661373761643766356366
          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
          3164
-```

-To enable Containerd **2+** to access the private registry:
-
-```yaml
-
-containerd_registries_mirrors:
-  - prefix: docker.io
-    mirrors:
-      - host: https://registry-1.docker.io
-        capabilities: ["pull", "resolve"]
-        skip_verify: false
-  - prefix: "{{ registry_addr }}"
-    mirrors:
-      - host: "https://{{ registry_addr }}"
-        capabilities: ["pull", "resolve"]
-        skip_verify: false
-        header:
-          Authorization: ["Basic {{ (registry_user + ':' + registry_pass) | b64encode }}"]
-```
-
-To enable Containerd **1.7** to access the private registry:
-
-```yaml
 containerd_registry_auth:
  - registry: "{{ registry_host }}"
    username: "{{ registry_user }}"
@@ -101,12 +73,25 @@ gcr_image_repo: "{{ registry_host }}"
 docker_image_repo: "{{ registry_host }}"
 quay_image_repo: "{{ registry_host }}"
 github_image_repo: "{{ registry_host }}"
-github_url: "{{ files_repo }}/github.com"
-dl_k8s_io_url: "{{ files_repo }}/dl.k8s.io"
-storage_googleapis_url: "{{ files_repo }}/storage.googleapis.com"
-get_helm_url: "{{ files_repo }}/get.helm.sh"
+
 local_path_provisioner_helper_image_repo: "{{ registry_host }}/busybox"
-# Insecure registries for containerd (see authenticated example above)
+kubeadm_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubeadm"
+kubectl_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubectl"
+kubelet_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubelet"
+# etcd is optional if you **DON'T** use etcd_deployment=host
+etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
+crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# If using Calico
+calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# If using Calico with kdd
+calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_version }}.tar.gz"
+# Containerd
+containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
+runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}"
+nerdctl_download_url: "{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+get_helm_url: "{{ files_repo }}/get.helm.sh"
+# Insecure registries for containerd
 containerd_registries_mirrors:
  - prefix: "{{ registry_addr }}"
    mirrors:
@@ -114,34 +99,6 @@ containerd_registries_mirrors:
        capabilities: ["pull", "resolve"]
        skip_verify: true

-# Cilium
-cilium_install_extra_flags: "--repository {{ files_repo }}/helm.cilium.io/"
-cilium_extra_values:
-  image:
-    useDigest: false
-  hubble:
-    relay:
-      image:
-        useDigest: false
-    ui:
-      backend:
-        image:
-          useDigest: false
-      frontend:
-        image:
-          useDigest: false
-  operator:
-    image:
-      override: "{{ registry_host }}/cilium/operator-generic:v1.18.2"
-      useDigest: false
-      extension: ""
-  certgen:
-    image:
-      useDigest: false
-  envoy:
-    image:
-      useDigest: false
-
 # CentOS/Redhat/AlmaLinux/Rocky Linux
 ## Docker / Containerd
 docker_rh_repo_base_url: "{{ yum_repo }}/docker-ce/$releasever/$basearch"
@@ -26,13 +26,13 @@ If you wanted to upgrade just kube_version from v1.18.10 to v1.19.7, you could
 deploy the following way:

 ```ShellSession
-ansible-playbook cluster.yml -i inventory/sample/inventory.ini -e kube_version=1.18.10 -e upgrade_cluster_setup=true
+ansible-playbook cluster.yml -i inventory/sample/hosts.ini -e kube_version=1.18.10 -e upgrade_cluster_setup=true
 ```

 And then repeat with 1.19.7 as kube_version:

 ```ShellSession
-ansible-playbook cluster.yml -i inventory/sample/inventory.ini -e kube_version=1.19.7 -e upgrade_cluster_setup=true
+ansible-playbook cluster.yml -i inventory/sample/hosts.ini -e kube_version=1.19.7 -e upgrade_cluster_setup=true
 ```

 The var ```-e upgrade_cluster_setup=true``` is needed to be set in order to migrate the deploys of e.g kube-apiserver inside the cluster immediately which is usually only done in the graceful upgrade. (Refer to [#4139](https://github.com/kubernetes-sigs/kubespray/issues/4139) and [#4736](https://github.com/kubernetes-sigs/kubespray/issues/4736))
@@ -46,7 +46,7 @@ existing cluster. That means there must be at least 1 kube_control_plane already
 deployed.

 ```ShellSession
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/inventory.ini -e kube_version=1.19.7
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.19.7
 ```

 After a successful upgrade, the Server Version should be updated:
@@ -60,7 +60,7 @@ Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7", GitCom
 You can control how many nodes are upgraded at the same time by modifying the ansible variable named `serial`, as explained [here](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_strategies.html#setting-the-batch-size-with-serial). If you don't set this variable, it will upgrade the cluster nodes in batches of  20% of the available nodes. Setting `serial=1` would mean upgrade one node at a time.

 ```ShellSession
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/inventory.ini -e kube_version=1.20.7 -e "serial=1"
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 -e "serial=1"
 ```

 ### Pausing the upgrade
@@ -82,20 +82,20 @@ If you don't want to upgrade all nodes in one run, you can use `--limit` [patter
 Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes:

 ```ShellSession
-ansible-playbook playbooks/facts.yml -b -i inventory/sample/inventory.ini
+ansible-playbook playbooks/facts.yml -b -i inventory/sample/hosts.ini
 ```

 After this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147):

 ```ShellSession
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/inventory.ini -e kube_version=1.20.7 --limit "kube_control_plane:etcd"
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit "kube_control_plane:etcd"
 ```

 Now you can upgrade other nodes in any order and quantity:

 ```ShellSession
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/inventory.ini -e kube_version=1.20.7 --limit "node4:node6:node7:node12"
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/inventory.ini -e kube_version=1.20.7 --limit "node5*"
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit "node4:node6:node7:node12"
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit "node5*"
 ```

 ## Multiple upgrades
@@ -122,9 +122,9 @@ v2.24.0
 v2.22.0 -> v2.23.2 -> v2.24.0 : ✓
 v.22.0 -> v2.24.0 : ✕

-Assuming you don't explicitly define a kubernetes version in your group_vars/k8s_cluster/k8s-cluster.yml, you simply check out the next tag and run the upgrade-cluster.yml playbook
+Assuming you don't explicitly define a kubernetes version in your k8s_cluster.yml, you simply check out the next tag and run the upgrade-cluster.yml playbook

-* If you do define kubernetes version in your inventory (e.g. group_vars/k8s_cluster/k8s-cluster.yml) then either make sure to update it before running upgrade-cluster, or specify the new version you're upgrading to: `ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml -e kube_version=1.11.3`
+* If you do define kubernetes version in your inventory (e.g. group_vars/k8s_cluster.yml) then either make sure to update it before running upgrade-cluster, or specify the new version you're upgrading to: `ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml -e kube_version=1.11.3`

  Otherwise, the upgrade will leave your cluster at the same k8s version defined in your inventory vars.

@@ -155,7 +155,7 @@ HEAD is now at 05dabb7e Fix Bionic networking restart error #3430 (#3431)

 # NOTE: May need to `pip3 install -r requirements.txt` when upgrading.

-ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml

 ...

@@ -178,7 +178,7 @@ Some deprecations between versions that mean you can't just upgrade straight fro
 In this case, I set "kubeadm_enabled" to false, knowing that it is deprecated and removed by 2.9.0, to delay converting the cluster to kubeadm as long as I could.

 ```ShellSession
-$ ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml
 ...
    "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release."
 ...
@@ -196,7 +196,7 @@ $ git checkout v2.8.1
 Previous HEAD position was 9051aa52 Fix ubuntu-contiv test failed (#3808)
 HEAD is now at 2ac1c756 More Feature/2.8 backports for 2.8.1 (#3911)

-$ ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml
 ...
    "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release."
 ...
@@ -214,7 +214,7 @@ $ git checkout v2.8.2
 Previous HEAD position was 2ac1c756 More Feature/2.8 backports for 2.8.1 (#3911)
 HEAD is now at 4167807f Upgrade to 1.12.5 (#4066)

-$ ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml
 ...
    "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release."
 ...
@@ -232,7 +232,7 @@ $ git checkout v2.8.3
 Previous HEAD position was 4167807f Upgrade to 1.12.5 (#4066)
 HEAD is now at ea41fc5e backport cve-2019-5736 to release-2.8 (#4234)

-$ ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml
 ...
    "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release."
 ...
@@ -250,7 +250,7 @@ $ git checkout v2.8.4
 Previous HEAD position was ea41fc5e backport cve-2019-5736 to release-2.8 (#4234)
 HEAD is now at 3901480b go to k8s 1.12.7 (#4400)

-$ ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml
 ...
    "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release."
 ...
@@ -268,7 +268,7 @@ $ git checkout v2.8.5
 Previous HEAD position was 3901480b go to k8s 1.12.7 (#4400)
 HEAD is now at 6f97687d Release 2.8 robust san handling (#4478)

-$ ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml
 ...
    "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release."
 ...
@@ -288,14 +288,14 @@ HEAD is now at a4e65c7c Upgrade to Ansible >2.7.0 (#4471)
 ```

 > **Warning**
-> IMPORTANT: Some variable formats changed in the group_vars/k8s_cluster/k8s-cluster.yml between 2.8.5 and 2.9.0
+> IMPORTANT: Some variable formats changed in the k8s_cluster.yml between 2.8.5 and 2.9.0

 If you do not keep your inventory copy up to date, **your upgrade will fail** and your first master will be left non-functional until fixed and re-run.

 It is at this point the cluster was upgraded from non-kubeadm to kubeadm as per the deprecation warning.

 ```ShellSession
-ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml

 ...

@@ -320,7 +320,7 @@ $ git checkout v2.10.0
 Previous HEAD position was a4e65c7c Upgrade to Ansible >2.7.0 (#4471)
 HEAD is now at dcd9c950 Add etcd role dependency on kube user to avoid etcd role failure when running scale.yml with a fresh node. (#3240) (#4479)

-ansible-playbook -i inventory/mycluster/inventory.ini -b upgrade-cluster.yml
+ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml

 ...

@@ -372,49 +372,49 @@ hosts.
 Upgrade docker:

 ```ShellSession
-ansible-playbook -b -i inventory/sample/inventory.ini cluster.yml --tags=docker
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=docker
 ```

 Upgrade etcd:

 ```ShellSession
-ansible-playbook -b -i inventory/sample/inventory.ini cluster.yml --tags=etcd
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd
 ```

 Upgrade etcd without rotating etcd certs:

 ```ShellSession
-ansible-playbook -b -i inventory/sample/inventory.ini cluster.yml --tags=etcd --limit=etcd --skip-tags=etcd-secrets
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd --limit=etcd --skip-tags=etcd-secrets
 ```

 Upgrade kubelet:

 ```ShellSession
-ansible-playbook -b -i inventory/sample/inventory.ini cluster.yml --tags=node --skip-tags=k8s-gen-certs
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=node --skip-tags=k8s-gen-certs
 ```

 Upgrade Kubernetes master components:

 ```ShellSession
-ansible-playbook -b -i inventory/sample/inventory.ini cluster.yml --tags=master
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=master
 ```

 Upgrade network plugins:

 ```ShellSession
-ansible-playbook -b -i inventory/sample/inventory.ini cluster.yml --tags=network
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=network
 ```

 Upgrade all add-ons:

 ```ShellSession
-ansible-playbook -b -i inventory/sample/inventory.ini cluster.yml --tags=apps
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=apps
 ```

 Upgrade just helm (assuming `helm_enabled` is true):

 ```ShellSession
-ansible-playbook -b -i inventory/sample/inventory.ini cluster.yml --tags=helm
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=helm
 ```

 ## Migrate from Docker to Containerd
@@ -430,7 +430,7 @@ As of Kubespray 2.18.0, containerd is already the default container engine. If y
 If you want to upgrade the APT or YUM packages while the nodes are cordoned, you can use:

 ```ShellSession
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/inventory.ini -e system_upgrade=true
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e system_upgrade=true
 ```

 Nodes will be rebooted when there are package upgrades (`system_upgrade_reboot: on-upgrade`).
@@ -28,11 +28,11 @@ Everything done here requires full root access to every node.
 Before you begin, adjust your inventory:

 ```yaml
-# Filename: group_vars/k8s_cluster/k8s-cluster.yml
+# Filename: k8s_cluster/k8s-cluster.yml
 resolvconf_mode: host_resolvconf
 container_manager: containerd

-# Filename: group_vars/all/etcd.yml
+# Filename: etcd.yml
 etcd_deployment_type: host
 ```

@@ -66,7 +66,7 @@ apt-get install pigz
 ### 5) Run `cluster.yml` playbook with `--limit`

 ```commandline
-ansible-playbook -i inventory/sample/inventory.ini cluster.yml --limit=NODENAME
+ansible-playbook -i inventory/sample/hosts.ini cluster.yml --limit=NODENAME
 ```

 This effectively reinstalls containerd and seems to place all config files in the right place. When this completes, kubelet will immediately pick up the new container engine and start spinning up DaemonSets and kube-system Pods.
@@ -2,7 +2,7 @@
 namespace: kubernetes_sigs
 description: Deploy a production ready Kubernetes cluster
 name: kubespray
-version: 2.32.0
+version: 2.29.1
 readme: README.md
 authors:
  - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray)
@@ -38,7 +38,6 @@
    loadSidebar: 'docs/_sidebar.md',
    repo: 'https://github.com/kubernetes-sigs/kubespray',
    auto2top: true,
-    noCompileLinks: ['.*\.ini'],
    logo: '/logo/logo-clear.png'
  }
 </script>
@@ -11,15 +11,15 @@
 # containerd_runc_runtime:
 #   name: runc
 #   type: "io.containerd.runc.v2"
-#   options:
-#     Root: ""
+#   engine: ""
+#   root: ""

 # containerd_additional_runtimes:
 # Example for Kata Containers as additional runtime:
 #   - name: kata
 #     type: "io.containerd.kata.v2"
-#     options:
-#       Root: ""
+#     engine: ""
+#     root: ""

 # containerd_grpc_max_recv_message_size: 16777216
 # containerd_grpc_max_send_message_size: 16777216
@@ -44,7 +44,7 @@
 # [Optional] Calico: If using Calico network plugin
 # calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
-# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/raw/v{{ calico_version }}/manifests/crds.yaml"
+# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/v{{ calico_version }}.tar.gz"

 # [Optional] Cilium: If using Cilium network plugin
 # ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
@@ -1,4 +1,8 @@
 ---
+# Kubernetes dashboard
+# RBAC required. see docs/getting-started.md for access details.
+# dashboard_enabled: false
+
 # Helm deployment
 helm_enabled: false

@@ -63,6 +67,39 @@ local_volume_provisioner_enabled: false
 # Gateway API CRDs
 gateway_api_enabled: false

+# Nginx ingress controller deployment
+ingress_nginx_enabled: false
+# ingress_nginx_host_network: false
+# ingress_nginx_service_type: LoadBalancer
+# ingress_nginx_service_annotations:
+#   example.io/loadbalancerIPs: 1.2.3.4
+# ingress_nginx_service_nodeport_http: 30080
+# ingress_nginx_service_nodeport_https: 30081
+ingress_publish_status_address: ""
+# ingress_nginx_nodeselector:
+#   kubernetes.io/os: "linux"
+# ingress_nginx_tolerations:
+#   - key: "node-role.kubernetes.io/control-plane"
+#     operator: "Equal"
+#     value: ""
+#     effect: "NoSchedule"
+# ingress_nginx_namespace: "ingress-nginx"
+# ingress_nginx_insecure_port: 80
+# ingress_nginx_secure_port: 443
+# ingress_nginx_configmap:
+#   map-hash-bucket-size: "128"
+#   ssl-protocols: "TLSv1.2 TLSv1.3"
+# ingress_nginx_configmap_tcp_services:
+#   9000: "default/example-go:8080"
+# ingress_nginx_configmap_udp_services:
+#   53: "kube-system/coredns:53"
+# ingress_nginx_extra_args:
+#   - --default-ssl-certificate=default/foo-tls
+# ingress_nginx_termination_grace_period_seconds: 300
+# ingress_nginx_class: nginx
+# ingress_nginx_without_class: true
+# ingress_nginx_default: false
+
 # ALB ingress controller deployment
 ingress_alb_enabled: false
 # alb_ingress_aws_region: "us-east-1"
@@ -199,10 +236,6 @@ kube_vip_enabled: false
 # kube_vip_leasename: plndr-cp-lock
 # kube_vip_enable_node_labeling: false
 # kube_vip_lb_fwdmethod: local
-# kube_vip_bgp_sourceip:
-# kube_vip_bgp_sourceif:
-# kube_vip_metrics_enabled: false
-# kube_vip_metrics_port: 2112

 # Node Feature Discovery
 node_feature_discovery_enabled: false
@@ -22,8 +22,7 @@ local_release_dir: "/tmp/releases"
 # Random shifts for retrying failed ops like pushing/downloading
 retry_stagger: 5

-# This is the user that owns the cluster installation.
-# Note: cilium needs to set kube_owner to root https://kubespray.io/#/docs/CNI/cilium?id=unprivileged-agent-configuration
+# This is the user that owns tha cluster installation.
 kube_owner: kube

 # This is the group that the cert creation scripts chgrp the
@@ -53,21 +52,6 @@ credentials_dir: "{{ inventory_dir }}/credentials"
 # kube_oidc_groups_claim: groups
 # kube_oidc_groups_prefix: 'oidc:'

-## Structured AuthenticationConfiguration https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-authentication-configuration
-## Note: --authentication-config and --oidc-* flags are mutually exclusive
-# kube_apiserver_use_authentication_config_file: false
-# kube_apiserver_authentication_config_jwt:
-#   - issuer:
-#       url: https://issuer.example.com
-#       audiences:
-#       - my-audience
-#     claimMappings:
-#       username:
-#         expression: 'claims.sub'
-# kube_apiserver_authentication_config_anonymous:
-#   enabled: "{{ kube_api_anonymous_auth }}"
-#   conditions: []
-
 ## Variables to control webhook authn/authz
 # kube_webhook_token_auth: false
 # kube_webhook_token_auth_url: https://...
@@ -56,8 +56,8 @@ cilium_l2announcements: false
 #
 # Only effective when monitor aggregation is set to "medium" or higher.
 # cilium_monitor_aggregation_flags: "all"
-# Kube Proxy Replacement mode (true/false)
-# cilium_kube_proxy_replacement: false
+# Kube Proxy Replacement mode (strict/partial)
+# cilium_kube_proxy_replacement: partial

 # If upgrading from Cilium < 1.5, you may want to override some of these options
 # to prevent service disruptions. See also:
@@ -361,6 +361,8 @@ cilium_l2announcements: false
 # -- Enable the use of well-known identities.
 # cilium_enable_well_known_identities: false

+# cilium_enable_bpf_clock_probe: true
+
 # -- Whether to enable CNP status updates.
 # cilium_disable_cnp_status_updates: true

@@ -1,2 +1,2 @@
 ---
-requires_ansible: ">=2.18.0,<2.19.0"
+requires_ansible: ">=2.17.3"
@@ -1,5 +1,5 @@
-# Use immutable image tags rather than mutable tags (like ubuntu:24.04)
-FROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b
+# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
+FROM ubuntu:jammy-20230308
 # Some tools like yamllint need this
 # Pip needs this as well at the moment to install ansible
 # (and potentially other packages)
@@ -27,14 +27,14 @@ RUN apt update -q \
         ca-certificates \
         curl \
         gnupg2 \
+         software-properties-common \
         unzip \
         libvirt-clients \
         qemu-utils \
         qemu-kvm \
         dnsmasq \
-        && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \
-        && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
-            $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | tee /etc/apt/sources.list.d/docker.list \
+    && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+    && add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
    && apt update -q \
    && apt install --no-install-recommends -yq docker-ce \
    && apt autoremove -yqq --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*
@@ -44,10 +44,11 @@ ADD ./requirements.txt  /kubespray/requirements.txt
 ADD ./tests/requirements.txt /kubespray/tests/requirements.txt

 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
-    && pip install --break-system-packages --ignore-installed --no-compile --no-cache-dir pip -U \
-    && pip install --break-system-packages --no-compile --no-cache-dir -r tests/requirements.txt \
-    && curl -L https://dl.k8s.io/release/v1.36.1/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && echo $(curl -L https://dl.k8s.io/release/v1.36.1/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && pip install --no-compile --no-cache-dir pip -U \
+    && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
+    && pip install --no-compile --no-cache-dir -r requirements.txt \
+    && curl -L https://dl.k8s.io/release/v1.33.7/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && echo $(curl -L https://dl.k8s.io/release/v1.33.7/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
    && chmod a+x /usr/local/bin/kubectl \
    # Install Vagrant
    && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
@@ -55,5 +56,5 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
    && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
    && vagrant plugin install vagrant-libvirt \
    # Install Kubernetes collections
-    && pip install --break-system-packages --no-compile --no-cache-dir kubernetes==35.0.0 \
-    && ansible-galaxy collection install kubernetes.core:==6.4.0
+    && pip install --no-compile --no-cache-dir kubernetes \
+    && ansible-galaxy collection install kubernetes.core
@@ -5,8 +5,8 @@
  become: false
  run_once: true
  vars:
-    minimal_ansible_version: 2.18.0
-    maximal_ansible_version: 2.19.0
+    minimal_ansible_version: 2.17.3
+    maximal_ansible_version: 2.18.0
  tags: always
  tasks:
    - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"
@@ -16,8 +16,6 @@
    - name: Gather and compute network facts
      import_role:
        name: network_facts
-      tags:
-        - always
    - name: Gather minimal facts
      setup:
        gather_subset: '!all'
@@ -55,7 +55,7 @@
    - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }
    - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
    - { role: kubernetes/node, tags: node }
-    - { role: kubernetes/control-plane, tags: control-plane, upgrade_cluster_setup: true }
+    - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }
    - { role: kubernetes/client, tags: client }
    - { role: kubernetes/node-label, tags: node-label }
    - { role: kubernetes/node-taint, tags: node-taint }
@@ -100,7 +100,7 @@
  environment: "{{ proxy_disable_env }}"
  roles:
    - { role: kubespray_defaults }
-    - { role: win_nodes/kubernetes_patch, tags: ["control-plane", "win_nodes"] }
+    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }

 - name: Install Calico Route Reflector
  hosts: calico_rr
@@ -1,7 +1,7 @@
-ansible==11.13.0
+ansible==10.7.0
 # Needed for community.crypto module
-cryptography==48.0.0
+cryptography==46.0.2
 # Needed for jinja2 json_query templating
-jmespath==1.1.0
+jmespath==1.0.1
 # Needed for ansible.utils.ipaddr
 netaddr==1.3.0
@@ -9,8 +9,6 @@ platforms:
    vm_memory: 512
 provisioner:
  name: ansible
-  env:
-    ANSIBLE_ROLES_PATH: ../../../
  config_options:
    defaults:
      callbacks_enabled: profile_tasks
@@ -1,2 +1,2 @@
 ---
-ssh_bastion_config_name: ssh-bastion.conf
+ssh_bastion_confing__name: ssh-bastion.conf
@@ -8,8 +8,8 @@
  tasks:
    - name: Copy config to remote host
      copy:
-        src: "{{ playbook_dir }}/{{ ssh_bastion_config_name }}"
-        dest: "{{ ssh_bastion_config_name }}"
+        src: "{{ playbook_dir }}/{{ ssh_bastion_confing__name }}"
+        dest: "{{ ssh_bastion_confing__name }}"
        owner: "{{ ansible_user }}"
        group: "{{ ansible_user }}"
        mode: "0644"
@@ -9,8 +9,6 @@ platforms:
    vm_memory: 512
 provisioner:
  name: ansible
-  env:
-    ANSIBLE_ROLES_PATH: ../../../
  config_options:
    defaults:
      callbacks_enabled: profile_tasks
@@ -17,6 +17,6 @@
  delegate_to: localhost
  connection: local
  template:
-    src: "{{ ssh_bastion_config_name }}.j2"
-    dest: "{{ playbook_dir }}/{{ ssh_bastion_config_name }}"
+    src: "{{ ssh_bastion_confing__name }}.j2"
+    dest: "{{ playbook_dir }}/{{ ssh_bastion_confing__name }}"
    mode: "0640"
@@ -12,18 +12,10 @@ coreos_locksmithd_disable: false
 # Install epel repo on Centos/RHEL
 epel_enabled: false

-## openEuler specific variables
-# Enable metalink for openEuler repos (auto-selects fastest mirror by location)
-openeuler_metalink_enabled: false
-
 ## Oracle Linux specific variables
 # Install public repo on Oracle Linux
 use_oracle_public_repo: true

-## RHEL specific variables
-# Enable RHEL subscription repos
-rhel_enable_repos: true
-
 ## Ubuntu specific variables
 # Disable unattended-upgrades for Linux kernel and all packages start with linux- on Ubuntu
 ubuntu_kernel_unattended_upgrades_disabled: false
@@ -21,8 +21,6 @@ platforms:
    vm_memory: 512
 provisioner:
  name: ansible
-  env:
-    ANSIBLE_ROLES_PATH: ../../../
  config_options:
    defaults:
      callbacks_enabled: profile_tasks
@@ -23,7 +23,7 @@
    name: "oracle-epel-release-el{{ ansible_distribution_major_version }}"
    state: present
  when:
-    - use_oracle_public_repo
+    - use_oracle_public_repo | default(true)
    - '''ID="ol"'' in os_release.stdout_lines'
    - (ansible_distribution_version | float) >= 7.6

@@ -39,7 +39,7 @@
    - { option: "enabled", value: "1" }
    - { option: "baseurl", value: "http://yum.oracle.com/repo/OracleLinux/OL{{ ansible_distribution_major_version }}/addons/$basearch/" }
  when:
-    - use_oracle_public_repo
+    - use_oracle_public_repo | default(true)
    - '''ID="ol"'' in os_release.stdout_lines'
    - (ansible_distribution_version | float) >= 7.6

@@ -56,7 +56,7 @@
    - { option: "gpgcheck", value: "0" }
    - { option: "baseurl", value: "http://mirror.centos.org/centos/{{ ansible_distribution_major_version }}/extras/$basearch/os/" }
  when:
-    - use_oracle_public_repo
+    - use_oracle_public_repo | default(true)
    - '''ID="ol"'' in os_release.stdout_lines'
    - (ansible_distribution_version | float) >= 7.6
    - (ansible_distribution_version | float) < 9
@@ -1,62 +1,3 @@
 ---
- name: Import CentOS bootstrap for openEuler
-  ansible.builtin.import_tasks: centos.yml
-
- name: Read openEuler release string
-  ansible.builtin.command: cat /etc/openEuler-release
-  register: _openeuler_release_raw
-  changed_when: false
-  check_mode: false
-  become: true
-  when: openeuler_metalink_enabled
-
-# /etc/openEuler-release contains e.g. "openEuler release 24.03 (LTS)"
-# The metalink server expects "24.03LTS" (no spaces/parens/dashes),
-# but DNF's $releasever resolves to just "24.03", causing "invalid repo or arch".
- name: Set openEuler metalink releasever
-  ansible.builtin.set_fact:
-    _openeuler_releasever: >-
-      {{ _openeuler_release_raw.stdout
-         | regex_replace('.*release\s+', '')
-         | regex_replace('[() -]', '') }}
-  when: openeuler_metalink_enabled
-
- name: Get existing openEuler repo sections
-  ansible.builtin.shell:
-    cmd: "set -o pipefail && grep '^\\[' /etc/yum.repos.d/openEuler.repo | tr -d '[]'"
-    executable: /bin/bash
-  register: _openeuler_repo_sections
-  changed_when: false
-  failed_when: false
-  check_mode: false
-  become: true
-  when: openeuler_metalink_enabled
-
- name: Enable metalink for openEuler repos
-  community.general.ini_file:
-    path: /etc/yum.repos.d/openEuler.repo
-    section: "{{ item.key }}"
-    option: metalink
-    value: "{{ item.value }}"
-    no_extra_spaces: true
-    mode: "0644"
-  loop: "{{ _openeuler_metalink_repos | dict2items | selectattr('key', 'in', _openeuler_repo_sections.stdout_lines | default([])) }}"
-  become: true
-  when: openeuler_metalink_enabled
-  register: _openeuler_metalink_result
-  vars:
-    _openeuler_metalink_repos:
-      OS: "https://mirrors.openeuler.org/metalink?repo={{ _openeuler_releasever }}/OS&arch=$basearch"
-      everything: "https://mirrors.openeuler.org/metalink?repo={{ _openeuler_releasever }}/everything&arch=$basearch"
-      EPOL: "https://mirrors.openeuler.org/metalink?repo={{ _openeuler_releasever }}/EPOL/main&arch=$basearch"
-      debuginfo: "https://mirrors.openeuler.org/metalink?repo={{ _openeuler_releasever }}/debuginfo&arch=$basearch"
-      source: "https://mirrors.openeuler.org/metalink?repo={{ _openeuler_releasever }}&arch=source"
-      update: "https://mirrors.openeuler.org/metalink?repo={{ _openeuler_releasever }}/update&arch=$basearch"
-      update-source: "https://mirrors.openeuler.org/metalink?repo={{ _openeuler_releasever }}/update&arch=source"
-
- name: Clean dnf metadata cache to apply metalink mirror selection
-  ansible.builtin.command: dnf clean metadata
-  become: true
-  when:
-    - openeuler_metalink_enabled
-    - _openeuler_metalink_result.changed
+- name: Import Centos boostrap for openEuler
+  import_tasks: centos.yml
@@ -69,7 +69,7 @@
    name:
      - "rhel-8-for-*-baseos-rpms"
      - "rhel-8-for-*-appstream-rpms"
-    state: "{{ 'enabled' if (rhel_enable_repos | bool) else 'disabled' }}"
+    state: "{{ 'enabled' if (rhel_enable_repos | default(True) | bool) else 'disabled' }}"
  when:
    - ansible_distribution_major_version == "8"
    - (not rh_subscription_status.changed) or (rh_subscription_username is defined) or (rh_subscription_org_id is defined)
@@ -13,9 +13,10 @@ containerd_snapshotter: "overlayfs"
 containerd_runc_runtime:
  name: runc
  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
  base_runtime_spec: cri-base.json
  options:
-    Root: ""
    SystemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
    BinaryName: "{{ bin_dir }}/runc"

@@ -23,8 +24,8 @@ containerd_additional_runtimes: []
 # Example for Kata Containers as additional runtime:
 #  - name: kata
 #    type: "io.containerd.kata.v2"
-#    options:
-#      Root: ""
+#    engine: ""
+#    root: ""

 containerd_base_runtime_spec_rlimit_nofile: 65535

@@ -35,8 +36,8 @@ containerd_default_base_runtime_spec_patch:
        hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
        soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}"

-# Only for containerd < 2.1; discard unpacked layers to save disk space
-# https://github.com/containerd/containerd/blob/release/2.1/docs/cri/config.md#image-pull-configuration-since-containerd-v21
+# Can help reduce disk usage
+# https://github.com/containerd/containerd/discussions/6295
 containerd_discard_unpacked_layers: true

 containerd_base_runtime_specs:
@@ -52,6 +52,8 @@ oom_score = {{ containerd_oom_score }}
 {% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
       [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.{{ runtime.name }}]
         runtime_type = "{{ runtime.type }}"
+         runtime_engine = "{{ runtime.engine }}"
+         runtime_root = "{{ runtime.root }}"
 {% if runtime.base_runtime_spec is defined %}
         base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
 {% endif %}
@@ -76,9 +78,7 @@ oom_score = {{ containerd_oom_score }}

  [plugins."io.containerd.cri.v1.images"]
    snapshotter = "{{ containerd_snapshotter }}"
-{% if containerd_discard_unpacked_layers and containerd_version is version('2.1.0', '<') %}
    discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}
-{% endif %}
    image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}"
  [plugins."io.containerd.cri.v1.images".pinned_images]
    sandbox = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
@@ -6,6 +6,12 @@
    masked: false
  listen: Restart and enable cri-dockerd

+- name: Cri-dockerd | restart docker.service
+  service:
+    name: docker.service
+    state: restarted
+  listen: Restart and enable cri-dockerd
+
 - name: Cri-dockerd | reload cri-dockerd.socket
  service:
    name: cri-dockerd.socket
@@ -25,8 +25,6 @@ provisioner:
    group_vars:
      all:
        become: true
-      k8s_cluster:
-        container_manager: docker
  playbooks:
    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
    prepare: ../../../molecule/prepare.yml
@@ -32,8 +32,6 @@ crio_registry_auth: []
 crio_seccomp_profile: ""
 crio_selinux: "{{ (preinstall_selinux_state == 'enforcing') | lower }}"
 crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}"
-# Set the pull progress timeout
-crio_pull_progress_timeout: "10s"

 # Override system default for storage driver
 # crio_storage_driver: "overlay"
@@ -2,6 +2,8 @@
 - name: Converge
  hosts: all
  become: true
+  vars:
+    container_manager: crio
  roles:
    - role: kubespray_defaults
    - role: container-engine/cri-o
@@ -41,10 +41,6 @@ provisioner:
    defaults:
      callbacks_enabled: profile_tasks
      timeout: 120
-  inventory:
-    group_vars:
-      k8s_cluster:
-        container_manager: crio
  playbooks:
    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
    prepare: ../../../molecule/prepare.yml
@@ -2,6 +2,7 @@
 - name: Test CRI-O cri
  import_playbook: ../../../molecule/test_cri.yml
  vars:
+    container_manager: crio
    cri_socket: unix:///var/run/crio/crio.sock
    cri_name: cri-o
 - name: Test running a container with crun
@@ -1,16 +1,16 @@
 {% if crio_registry_auth is defined and crio_registry_auth|length %}
 {
-  "auths": {
 {% for reg in crio_registry_auth %}
+  "auths": {
    "{{ reg.registry }}": {
      "auth": "{{ (reg.username + ':' + reg.password) | string | b64encode }}"
-{% if not loop.last %}
-    },
-{% else %}
    }
+{% if not loop.last %}
+  },
+{% else %}
+  }
 {% endif %}
 {% endfor %}
-  }
 }
 {% else %}
 {}
@@ -348,12 +348,6 @@ signature_policy = "{{ crio_signature_policy }}"
 # ignore; the latter will ignore volumes entirely.
 image_volumes = "mkdir"

-# The timeout for an image pull to make progress until the pull operation gets
-# canceled. This value will be also used for calculating the pull progress interval
-# to pull_progress_timeout / 10. Can be set to 0 to disable the timeout as well as
-# the progress output.
-pull_progress_timeout = "{{ crio_pull_progress_timeout }}"
-
 # The crio.network table containers settings pertaining to the management of
 # CNI plugins.
 [crio.network]
@@ -0,0 +1,22 @@
+---
+- name: Crictl | Download crictl
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crictl) }}"
+
+- name: Install crictl config
+  template:
+    src: crictl.yaml.j2
+    dest: /etc/crictl.yaml
+    owner: root
+    mode: "0644"
+
+- name: Copy crictl binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/crictl"
+    dest: "{{ bin_dir }}/crictl"
+    mode: "0755"
+    remote_src: true
+  notify:
+    - Get crictl completion
+    - Install crictl completion
@@ -1,22 +1,3 @@
 ---
- name: Crictl | Download crictl
-  include_tasks: "../../../download/tasks/download_file.yml"
-  vars:
-    download: "{{ download_defaults | combine(downloads.crictl) }}"
-
- name: Install crictl config
-  template:
-    src: crictl.yaml.j2
-    dest: /etc/crictl.yaml
-    owner: root
-    mode: "0644"
-
- name: Copy crictl binary from download dir
-  copy:
-    src: "{{ local_release_dir }}/crictl"
-    dest: "{{ bin_dir }}/crictl"
-    mode: "0755"
-    remote_src: true
-  notify:
-    - Get crictl completion
-    - Install crictl completion
+- name: Install crictl
+  include_tasks: crictl.yml
@@ -55,7 +55,7 @@
  register: keyserver_task_result
  until: keyserver_task_result is succeeded
  retries: 4
-  delay: "{{ retry_stagger }}"
+  delay: "{{ retry_stagger | d(3) }}"
  with_items: "{{ docker_repo_key_info.repo_keys }}"
  environment: "{{ proxy_env }}"
  when: ansible_pkg_mgr == 'apt'
@@ -128,7 +128,7 @@
  register: docker_task_result
  until: docker_task_result is succeeded
  retries: 4
-  delay: "{{ retry_stagger }}"
+  delay: "{{ retry_stagger | d(3) }}"
  notify: Restart docker
  when:
    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
@@ -30,7 +30,7 @@ LimitCORE=infinity
 TimeoutStartSec=1min
 # restart the docker process if it exits prematurely
 Restart=on-failure
-StartLimitBurst=10
+StartLimitBurst=3
 StartLimitInterval=60s
 # Set the cgroup slice of the service so that kube reserved takes effect
 {% if kube_reserved is defined and kube_reserved|bool %}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
ChengHao Yang	0c6a29553f	Patch versions updates (#12782 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-12-11 00:55:31 -08:00
Max Gautier	2375fae1c2	Patch versions updates (#12763 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-12-04 06:39:01 -08:00
Max Gautier	55f7b7f54c	Patch versions updates (#12744 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-25 00:52:36 -08:00
k8s-infra-cherrypick-robot	dbca6a7757	[release-2.29] CI: enable unsafe_show_logs == true by default (#12728 ) * CI: enable unsafe_show_logs == true by default * Deduplicate defaults vars (unsafe_show_logs) --------- Co-authored-by: Max Gautier <mg@max.gautier.name>	2025-11-19 23:32:02 -08:00
k8s-infra-cherrypick-robot	c5c43619a7	Upgrade cilium from 1.18.3 to 1.18.4 (#12725 ) Signed-off-by: Ali Afsharzadeh <afsharzadeh8@gmail.com> Co-authored-by: Ali Afsharzadeh <afsharzadeh8@gmail.com>	2025-11-18 20:09:59 -08:00
Max Gautier	584b0a4036	Patch versions updates (#12719 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-18 05:15:39 -08:00
k8s-infra-cherrypick-robot	084c2be8b9	CI: use a dedicated disk for releases (#12721 ) This should make 'no space left on device' problems easier to handle Use /tmp/releases as local_release_dir CI created machine, while keeping the same folder on the runner (needed for gitlab-ci runner pods) Co-authored-by: Max Gautier <mg@max.gautier.name>	2025-11-18 03:29:39 -08:00
k8s-infra-cherrypick-robot	932025fbd6	Let containerd create storage / state dir (#12722 ) Containerd manages by itself, so there is no need to override it and change permissions. Co-authored-by: Max Gautier <mg@max.gautier.name>	2025-11-18 03:09:38 -08:00
k8s-infra-cherrypick-robot	a04592de18	Adjust hubble export values for cilium 1.18 schema change (#12718 ) Signed-off-by: Ali Afsharzadeh <afsharzadeh8@gmail.com> Co-authored-by: Ali Afsharzadeh <afsharzadeh8@gmail.com>	2025-11-18 00:41:42 -08:00
k8s-infra-cherrypick-robot	d8b9288b27	[release-2.29] CI: Try a full ssh connection on hosts instead of only checking the port (#12711 ) * CI: Try a full ssh connection on hosts instead of only checking the port If we only try the port, we can try to connect in the playbook which is executed next even though the managed node has not yet completed it's boot-up sequence ("System is booting up. Unprivileged users are not permitted to log in yet. Please come back later. For technical details, see pam_nologin(8).") This does not account for python-less hosts, but we don't use those in CI anyway (for now, at least). * CI: Remove connection method override when creating VMs This prevented wait_for_connection to work correctly by hijacking the connection to localhost, thus bypassing the connection check. --------- Co-authored-by: Max Gautier <mg@max.gautier.name>	2025-11-15 12:37:38 -08:00
Max Gautier	cbdd7cf3a7	update pre-commit hooks (#12706 )	2025-11-14 22:41:40 -08:00
k8s-infra-cherrypick-robot	3c0cff983d	fix(cilium):correct loadBalancer.mode rendering in values.yaml (#12705 ) Co-authored-by: Anurag Ojha <aojharaj2004@gmail.com>	2025-11-14 07:01:40 -08:00
k8s-infra-cherrypick-robot	e5a1f68a2c	Update Calico apiserver RBAC for Kubernetes 1.33+ (#12695 ) Add missing RBAC permissions for Calico apiserver to function correctly with Kubernetes 1.33+ Changes: 1. Add K8s 1.33 ValidatingAdmissionPolicy resources to calico-webhook-reader - validatingadmissionpolicies - validatingadmissionpolicybindings Kubernetes 1.33 introduced ValidatingAdmissionPolicy resources (KEP-3488) that require explicit RBAC permissions. Without these changes, Calico apiserver on k8s 1.33+ will not work and needless errors are logged Co-authored-by: rickerc <chris.ricker@gmail.com>	2025-11-14 04:49:38 -08:00
k8s-infra-cherrypick-robot	fe566df651	Fix the (upgrade/remove_node) + collection test cases (#12687 ) The 'old' playbook and the collection use '-' and '_' as separator, which breaks the logic in scripts/testcases_run.sh. Add aliases using the old schemes to make the test work and avoid breaking anything. Both '-' and '_' variants will be deleted once we switch to supporting collection only. Co-authored-by: Max Gautier <mg@max.gautier.name>	2025-11-10 06:46:57 -08:00
k8s-infra-cherrypick-robot	59b3c686a8	[release-2.29] Remove etcd member by peerURLs (#12685 ) * Remove etcd member by peerURLs The way to obtain the IP of a particular member is convoluted and depend on multiple variables. The match is also textual and it's not clear against what we're matching It's also broken for etcd member which are not also Kubernetes nodes, because the "Lookup node IP in kubernetes" task will fail and abort the play. Instead, match against 'peerURLs', which does not need new variable, and use json output. * Add testcase for etcd removal on external etcd * do not merge * fixup! Remove etcd member by peerURLs * fixup! Remove etcd member by peerURLs --------- Co-authored-by: Max Gautier <mg@max.gautier.name>	2025-11-10 05:48:56 -08:00
Ali Afsharzadeh	4b970baa5a	[release-2.29] Upgrade cilium from 1.18.2 to 1.18.3 (#12679 )	2025-11-09 06:00:52 -08:00
ChengHao Yang	a15fcb729b	Patch versions updates (#12646 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-03 02:19:36 -08:00
k8s-infra-cherrypick-robot	9a9e33dc9f	fix(calico): Add missed rbac verb for hostendpoints (#12644 ) Signed-off-by: Meza <meza-xyz@proton.me> Co-authored-by: Meza <meza-xyz@proton.me>	2025-10-24 01:05:34 -07:00
ChengHao Yang	d9f188c39c	[release-2.29] Releng: galaxy version to 2.29.1 (#12645 ) Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>	2025-10-24 00:41:36 -07:00