Compare commits

..

28 Commits

Author SHA1 Message Date
Aleksandr Didenko 62a1925664 Fix in network names 2016-06-20 15:03:24 +02:00
Aleksandr Didenko 7f247754f9 Add network names 2016-06-20 15:02:11 +02:00
Aleksandr Didenko 7a53c32a3b Fix networks and add some debug tools
- Create two networks: public (NATed) and private (isolated)
- Add some debug tools to minion nodes
2016-06-20 14:43:20 +02:00
Aleksandr Didenko 0e48ce51ce Multiple fixes to deployment scripts and lab
- Clean up private/pub keys from k8s nodes
- Install kpm on k8s nodes, not on master node
2016-06-20 12:02:25 +02:00
Aleksandr Didenko bc29db7bd2 Fixes and improvements
- Move kubedns to a separate mini playbook
- Fix custom.yaml location
2016-06-20 11:06:01 +02:00
Aleksandr Didenko 8d8622bbb9 Switch to ansible-2.1 2016-06-20 09:58:09 +02:00
Aleksandr Didenko 943edb6dd2 Fix permissions and kargo custom.yaml 2016-06-20 09:13:07 +02:00
Aleksandr Didenko c81457c617 Update README 2016-06-16 17:50:44 +02:00
Aleksandr Didenko 4c6f85b8ae Start counting k8s nodes from 2
First node is our master node.
2016-06-16 14:42:53 +02:00
Aleksandr Didenko 60fa68e5f7 Bugfix: return missing deploy command 2016-06-16 14:36:07 +02:00
Aleksandr Didenko ea5b40ae0e Fix ssh config 2016-06-16 14:28:29 +02:00
Aleksandr Didenko daf02e029c Fix nodes list 2016-06-16 13:00:51 +02:00
Aleksandr Didenko 3824493b1d Added README 2016-06-16 12:36:58 +02:00
Aleksandr Didenko cdfbcc1046 Remove SSH keys and generate them instead 2016-06-16 12:29:33 +02:00
Aleksandr Didenko 62e98bd4b0 Removing deprecated stuff 2016-06-16 12:19:27 +02:00
Aleksandr Didenko 43b2b5b464 Bugfixing for newly added code with kargo-clu support 2016-06-16 12:00:18 +02:00
Aleksandr Didenko 531f611ea3 Add support for deployment via kargo-cli 2016-06-16 11:49:17 +02:00
Aleksandr Didenko b9ed54812b Remove controlled by puppet line 2016-06-15 11:50:09 +02:00
Aleksandr Didenko 338749be16 Added README for master node with usefill commands 2016-06-15 11:48:50 +02:00
Aleksandr Didenko a2f3048e7a Added some packages to bootstrap script 2016-06-15 10:26:44 +02:00
Aleksandr Didenko b009ca3ff8 Fix inv dir copy command 2016-06-14 19:10:20 +02:00
Aleksandr Didenko c714660c0b Updates to use new inventory settings 2016-06-14 19:04:09 +02:00
Aleksandr Didenko 70519e2c5a Refactor inventroty for the lab 2016-06-14 19:02:27 +02:00
Aleksandr Didenko d365fab9ec Add inventory.cfg download to bootstrap script 2016-06-14 18:51:43 +02:00
Aleksandr Didenko 785169b934 Added inventory.cfg for the lab 2016-06-14 18:49:31 +02:00
Aleksandr Didenko 48e2062d92 Added apt-get commands to bootstrap-node 2016-06-14 18:40:41 +02:00
Aleksandr Didenko d894529f07 Update ssh/config on master node 2016-06-14 18:30:30 +02:00
Oleksandr Didenko 7e08800876 Initial commit 2016-06-14 18:21:56 +02:00
1335 changed files with 191 additions and 125073 deletions
-43
View File
@@ -1,43 +0,0 @@
---
parseable: true
skip_list:
# see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules
# DO NOT add any other rules to this skip_list, instead use local `# noqa` with a comment explaining WHY it is necessary
# These rules are intentionally skipped:
#
# [role-name] "meta/main.yml" Role name role-name does not match ``^+$`` pattern
# Meta roles in Kubespray don't need proper names
# (Disabled in June 2021)
- 'role-name'
# [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
# In Kubespray we use variables that use camelCase to match their k8s counterparts
# (Disabled in June 2021)
- 'var-naming'
# [fqcn-builtins]
# Roles in kubespray don't need fully qualified collection names
# (Disabled in Feb 2023)
- 'fqcn-builtins'
# We use template in names
- 'name[template]'
# No changed-when on commands
# (Disabled in June 2023 after ansible upgrade; FIXME)
- 'no-changed-when'
# Disable run-once check with free strategy
# (Disabled in June 2023 after ansible upgrade; FIXME)
- 'run-once[task]'
exclude_paths:
# Generated files
- tests/files/custom_cni/cilium.yaml
- venv
- .github
- .ansible
- .cache
mock_modules:
- gluster.gluster.gluster_volume
-8
View File
@@ -1,8 +0,0 @@
# This file contains ignores rule violations for ansible-lint
inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml jinja[spacing]
roles/kubernetes/control-plane/defaults/main/kube-proxy.yml jinja[spacing]
roles/kubernetes/control-plane/defaults/main/main.yml jinja[spacing]
roles/kubernetes/kubeadm/defaults/main.yml jinja[spacing]
roles/kubernetes/node/defaults/main.yml jinja[spacing]
roles/kubernetes/preinstall/defaults/main.yml jinja[spacing]
roles/kubespray-defaults/defaults/main/main.yml jinja[spacing]
-15
View File
@@ -1,15 +0,0 @@
root = true
[*.{yaml,yml,yml.j2,yaml.j2}]
indent_style = space
indent_size = 2
trim_trailing_whitespace = true
insert_final_newline = true
charset = utf-8
[{Dockerfile}]
indent_style = space
indent_size = 2
trim_trailing_whitespace = true
insert_final_newline = true
charset = utf-8
-1
View File
@@ -1 +0,0 @@
docs/_sidebar.md linguist-generated=true
-148
View File
@@ -1,148 +0,0 @@
---
name: Bug Report
description: Report a bug encountered while using Kubespray
labels: kind/bug
body:
- type: markdown
attributes:
value: |
Please, be ready for followup questions, and please respond in a timely
manner. If we can't reproduce a bug or think a feature already exists, we
might close your issue. If we're wrong, PLEASE feel free to reopen it and
explain why.
- type: textarea
id: problem
attributes:
label: What happened?
description: |
Please provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner.
validations:
required: true
- type: textarea
id: expected
attributes:
label: What did you expect to happen?
validations:
required: true
- type: textarea
id: repro
attributes:
label: How can we reproduce it (as minimally and precisely as possible)?
validations:
required: true
- type: markdown
attributes:
value: '### Environment'
- type: dropdown
id: os
attributes:
label: OS
options:
- 'RHEL 9'
- 'RHEL 8'
- 'Fedora 40'
- 'Ubuntu 24'
- 'Ubuntu 22'
- 'Ubuntu 20'
- 'Debian 12'
- 'Debian 11'
- 'Flatcar Container Linux'
- 'openSUSE Leap'
- 'openSUSE Tumbleweed'
- 'Oracle Linux 9'
- 'Oracle Linux 8'
- 'AlmaLinux 9'
- 'AlmaLinux 8'
- 'Rocky Linux 9'
- 'Rocky Linux 8'
- 'Amazon Linux 2'
- 'Kylin Linux Advanced Server V10'
- 'UOS Linux 20'
- 'openEuler 24'
- 'openEuler 22'
- 'openEuler 20'
- 'Other|Unsupported'
validations:
required: true
- type: textarea
id: ansible_version
attributes:
label: Version of Ansible
placeholder: 'ansible --version'
validations:
required: true
- type: input
id: python_version
attributes:
label: Version of Python
placeholder: 'python --version'
validations:
required: true
- type: input
id: kubespray_version
attributes:
label: Version of Kubespray (commit)
placeholder: 'git rev-parse --short HEAD'
validations:
required: true
- type: dropdown
id: network_plugin
attributes:
label: Network plugin used
options:
- calico
- cilium
- cni
- custom_cni
- flannel
- kube-ovn
- kube-router
- macvlan
- meta
- multus
- ovn4nfv
- weave
validations:
required: true
- type: textarea
id: inventory
attributes:
label: Full inventory with variables
placeholder: 'ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"'
description: We recommend using snippets services like https://gist.github.com/ etc.
validations:
required: true
- type: input
id: ansible_command
attributes:
label: Command used to invoke ansible
validations:
required: true
- type: textarea
id: ansible_output
attributes:
label: Output of ansible run
description: We recommend using snippets services like https://gist.github.com/ etc.
validations:
required: true
- type: textarea
id: anything_else
attributes:
label: Anything else we need to know
description: |
By running scripts/collect-info.yaml you can get a lot of useful informations.
Script can be started by:
ansible-playbook -i <inventory_file_path> -u <ssh_user> -e ansible_ssh_user=<ssh_user> -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml
(If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python').
After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here
-6
View File
@@ -1,6 +0,0 @@
---
blank_issues_enabled: false
contact_links:
- name: Support Request
url: https://kubernetes.slack.com/channels/kubespray
about: Support request or question relating to Kubernetes
-20
View File
@@ -1,20 +0,0 @@
---
name: Enhancement Request
description: Suggest an enhancement to the Kubespray project
labels: kind/feature
body:
- type: markdown
attributes:
value: Please only use this template for submitting enhancement requests
- type: textarea
id: what
attributes:
label: What would you like to be added
validations:
required: true
- type: textarea
id: why
attributes:
label: Why is this needed
validations:
required: true
-41
View File
@@ -1,41 +0,0 @@
---
name: Failing Test
description: Report test failures in Kubespray CI jobs
labels: kind/failing-test
body:
- type: markdown
attributes:
value: Please only use this template for submitting reports about failing tests in Kubespray CI jobs
- type: textarea
id: failing_jobs
attributes:
label: Which jobs are failing ?
validations:
required: true
- type: textarea
id: failing_tests
attributes:
label: Which tests are failing ?
validations:
required: true
- type: input
id: since_when
attributes:
label: Since when has it been failing ?
validations:
required: true
- type: textarea
id: failure_reason
attributes:
label: Reason for failure
description: If you don't know and have no guess, just put "Unknown"
validations:
required: true
- type: textarea
id: anything_else
attributes:
label: Anything else we need to know
-44
View File
@@ -1,44 +0,0 @@
<!-- Thanks for sending a pull request! Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md and developer guide https://git.k8s.io/community/contributors/devel/development.md
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. Follow the instructions for writing a release note: https://git.k8s.io/community/contributors/guide/release-notes.md
6. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->
**What type of PR is this?**
> Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line:
>
> /kind api-change
> /kind bug
> /kind cleanup
> /kind design
> /kind documentation
> /kind failing-test
> /kind feature
> /kind flake
**What this PR does / why we need it**:
**Which issue(s) this PR fixes**:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #
**Special notes for your reviewer**:
**Does this PR introduce a user-facing change?**:
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".
-->
```release-note
```
-20
View File
@@ -1,20 +0,0 @@
version: 2
updates:
- package-ecosystem: "pip"
directory: "/"
schedule:
interval: "weekly"
labels:
- dependencies
- release-note-none
groups:
molecule:
patterns:
- molecule
- molecule-plugins*
- package-ecosystem: "github-actions"
directory: "/"
labels:
- release-note-none
schedule:
interval: "weekly"
-32
View File
@@ -1,32 +0,0 @@
name: Issue labeler
on:
issues:
types: [opened]
permissions:
contents: read
jobs:
label-component:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- uses: actions/checkout@v3
- name: Parse issue form
uses: stefanbuck/github-issue-parser@v3
id: issue-parser
with:
template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
- name: Set labels based on OS field
uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
with:
issue-form: ${{ steps.issue-parser.outputs.jsonString }}
section: os
block-list: |
None
Other
token: ${{ secrets.GITHUB_TOKEN }}
@@ -1,54 +0,0 @@
name: Upgrade Kubespray components with new patches versions - all branches
on:
schedule:
- cron: '22 2 * * *' # every day, 02:22 UTC
workflow_dispatch:
permissions: {}
jobs:
get-releases-branches:
runs-on: ubuntu-latest
outputs:
branches: ${{ steps.get-branches.outputs.data }}
steps:
- uses: octokit/graphql-action@v2.3.2
id: get-branches
with:
query: |
query get_release_branches($owner:String!, $name:String!) {
repository(owner:$owner, name:$name) {
refs(refPrefix: "refs/heads/",
first: 0, # TODO increment once we have release branch with the new checksums format
query: "release-",
orderBy: {
field: ALPHABETICAL,
direction: DESC
}) {
nodes {
name
}
}
}
}
variables: |
owner: ${{ github.repository_owner }}
name: ${{ github.event.repository.name }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
update-versions:
needs: get-releases-branches
strategy:
fail-fast: false
matrix:
branch:
- name: ${{ github.event.repository.default_branch }}
- ${{ fromJSON(needs.get-releases-branches.outputs.branches).repository.refs.nodes }}
uses: ./.github/workflows/upgrade-patch-versions.yml
permissions:
contents: write
pull-requests: write
name: Update patch updates on ${{ matrix.branch.name }}
with:
branch: ${{ matrix.branch.name }}
@@ -1,44 +0,0 @@
on:
workflow_call:
inputs:
branch:
description: Which branch to update with new patch versions
default: master
required: true
type: string
jobs:
update-patch-versions:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.branch }}
- uses: actions/setup-python@v5
with:
python-version: '3.13'
cache: 'pip'
- run: pip install scripts/component_hash_update pre-commit
- run: update-hashes
env:
API_KEY: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v4
with:
key: pre-commit-hook-propagate
path: |
~/.cache/pre-commit
- run: pre-commit run --all-files propagate-ansible-variables
continue-on-error: true
- uses: peter-evans/create-pull-request@v7
with:
commit-message: Patch versions updates
title: Patch versions updates - ${{ inputs.branch }}
labels: bot
branch: ${{ inputs.branch }}-patch-updates
sign-commits: true
body: |
/kind feature
```release-note
NONE
```
+1 -122
View File
@@ -1,122 +1 @@
.vagrant
*.retry
**/vagrant_ansible_inventory
*.iml
temp
contrib/offline/container-images
contrib/offline/container-images.tar.gz
contrib/offline/offline-files
contrib/offline/offline-files.tar.gz
.idea
.vscode
.tox
.cache
*.bak
*.tfstate
*.tfstate*backup
*.lock.hcl
.terraform/
contrib/terraform/aws/credentials.tfvars
.terraform.lock.hcl
/ssh-bastion.conf
**/*.sw[pon]
*~
vagrant/
plugins/mitogen
# Ansible inventory
inventory/*
!inventory/local
!inventory/sample
inventory/*/artifacts/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
env/
build/
credentials/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# IPython Notebook
.ipynb_checkpoints
# pyenv
.python-version
# dotenv
.env
# virtualenv
venv/
ENV/
# molecule
roles/**/molecule/**/__pycache__/
# macOS
.DS_Store
# Temp location used by our scripts
scripts/tmp/
tmp.md
# Ansible collection files
kubernetes_sigs-kubespray*tar.gz
ansible_collections
ssh
-107
View File
@@ -1,107 +0,0 @@
---
stages:
- build
- test
- deploy-part1
- deploy-extended
variables:
FAILFASTCI_NAMESPACE: 'kargo-ci'
GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
GIT_CONFIG_COUNT: 2
GIT_CONFIG_KEY_0: user.email
GIT_CONFIG_VALUE_0: "ci@kubespray.io"
GIT_CONFIG_KEY_1: user.name
GIT_CONFIG_VALUE_1: "Kubespray CI"
ANSIBLE_FORCE_COLOR: "true"
MAGIC: "ci check this"
GS_ACCESS_KEY_ID: $GS_KEY
GS_SECRET_ACCESS_KEY: $GS_SECRET
CONTAINER_ENGINE: docker
GCE_PREEMPTIBLE: "false"
ANSIBLE_KEEP_REMOTE_FILES: "1"
ANSIBLE_CONFIG: ./tests/ansible.cfg
ANSIBLE_REMOTE_USER: kubespray
ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
ANSIBLE_INVENTORY: /tmp/inventory
RESET_CHECK: "false"
REMOVE_NODE_CHECK: "false"
UPGRADE_TEST: "false"
MITOGEN_ENABLE: "false"
ANSIBLE_VERBOSITY: 2
RECOVER_CONTROL_PLANE_TEST: "false"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
TERRAFORM_VERSION: 1.3.7
PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
before_script:
- ./tests/scripts/rebase.sh
- mkdir -p /.ssh
.job: &job
tags:
- ffci
image: $PIPELINE_IMAGE
artifacts:
when: always
paths:
- cluster-dump/
needs:
- pipeline-image
variables:
ANSIBLE_STDOUT_CALLBACK: "debug"
.job-moderated:
extends: .job
needs:
- pipeline-image
- ci-not-authorized
- pre-commit # lint
- vagrant-validate # lint
.testcases: &testcases
extends: .job-moderated
interruptible: true
before_script:
- update-alternatives --install /usr/bin/python python /usr/bin/python3 1
- ./tests/scripts/rebase.sh
- ./tests/scripts/testcases_prepare.sh
script:
- ./tests/scripts/testcases_run.sh
after_script:
- ./tests/scripts/testcases_cleanup.sh
# For failfast, at least 1 job must be defined in .gitlab-ci.yml
# Premoderated with manual actions
ci-not-authorized:
stage: build
before_script: []
after_script: []
rules:
# LGTM or ok-to-test labels
- if: $PR_LABELS =~ /.*,(lgtm|approved|ok-to-test).*|^(lgtm|approved|ok-to-test).*/i
variables:
CI_OK_TO_TEST: '0'
when: always
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
variables:
CI_OK_TO_TEST: '0'
- if: $CI_COMMIT_BRANCH == "master"
variables:
CI_OK_TO_TEST: '0'
- when: always
variables:
CI_OK_TO_TEST: '1'
script:
- exit $CI_OK_TO_TEST
tags:
- ffci
needs: []
include:
- .gitlab-ci/build.yml
- .gitlab-ci/lint.yml
- .gitlab-ci/terraform.yml
- .gitlab-ci/packet.yml
- .gitlab-ci/vagrant.yml
- .gitlab-ci/molecule.yml
-33
View File
@@ -1,33 +0,0 @@
---
.build-container:
cache:
key: $CI_COMMIT_REF_SLUG
paths:
- image-cache
tags:
- ffci
stage: build
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: ['']
variables:
TAG: $CI_COMMIT_SHORT_SHA
PROJECT_DIR: $CI_PROJECT_DIR
DOCKERFILE: Dockerfile
GODEBUG: "http2client=0"
before_script:
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
script:
- /kaniko/executor --cache=true
--cache-dir=image-cache
--context $PROJECT_DIR
--dockerfile $PROJECT_DIR/$DOCKERFILE
--label 'git-branch'=$CI_COMMIT_REF_SLUG
--label 'git-tag=$CI_COMMIT_TAG'
--destination $PIPELINE_IMAGE
--log-timestamp=true
pipeline-image:
extends: .build-container
variables:
DOCKERFILE: pipeline.Dockerfile
-26
View File
@@ -1,26 +0,0 @@
---
pre-commit:
stage: test
tags:
- ffci
image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
variables:
PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
script:
- pre-commit run --all-files --show-diff-on-failure
cache:
key: pre-commit-2
paths:
- ${PRE_COMMIT_HOME}
when: 'always'
needs: []
vagrant-validate:
extends: .job
stage: test
tags: [ffci]
variables:
VAGRANT_VERSION: 2.3.7
script:
- ./tests/scripts/vagrant-validate.sh
except: ['triggers', 'master']
-56
View File
@@ -1,56 +0,0 @@
---
.molecule:
tags: [ffci]
only: [/^pr-.*$/]
except: ['triggers']
stage: deploy-part1
image: $PIPELINE_IMAGE
needs:
- pipeline-image
# - ci-not-authorized
before_script:
- ./tests/scripts/rebase.sh
script:
- ./tests/scripts/molecule_run.sh
after_script:
- ./tests/scripts/molecule_logs.sh
artifacts:
when: always
paths:
- molecule_logs/
molecule:
extends: .molecule
script:
- ./tests/scripts/molecule_run.sh -i $ROLE
parallel:
matrix:
- ROLE:
- container-engine/cri-dockerd
- container-engine/containerd
- container-engine/cri-o
- adduser
- bastion-ssh-config
- bootstrap-os
# CI template for periodic CI jobs
# Enabled when PERIODIC_CI_ENABLED var is set
molecule_full:
only:
variables:
- $PERIODIC_CI_ENABLED
allow_failure: true
extends: molecule
parallel:
matrix:
- ROLE:
- container-engine/cri-dockerd
- container-engine/containerd
- container-engine/cri-o
- adduser
- bastion-ssh-config
- bootstrap-os
# FIXME : tests below are perma-failing
- container-engine/kata-containers
- container-engine/gvisor
- container-engine/youki
-257
View File
@@ -1,257 +0,0 @@
---
.packet:
extends: .testcases
variables:
ANSIBLE_TIMEOUT: "120"
CI_PLATFORM: packet
SSH_USER: kubespray
tags:
- ffci
needs:
- pipeline-image
- ci-not-authorized
# CI template for PRs
.packet_pr:
stage: deploy-part1
rules:
- if: $PR_LABELS =~ /.*ci-short.*/
when: manual
allow_failure: true
- if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
when: on_success
- when: manual
allow_failure: true
extends: .packet
## Uncomment this to have multiple stages
# needs:
# - packet_ubuntu20-calico-all-in-one
.packet_pr_short:
stage: deploy-part1
extends: .packet
rules:
- if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
when: on_success
- when: manual
allow_failure: true
.packet_pr_manual:
extends: .packet_pr
stage: deploy-extended
rules:
- if: $PR_LABELS =~ /.*ci-full.*/
when: on_success
# Else run as manual
- when: manual
allow_failure: true
.packet_pr_extended:
extends: .packet_pr
stage: deploy-extended
rules:
- if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
when: on_success
- when: manual
allow_failure: true
# CI template for periodic CI jobs
# Enabled when PERIODIC_CI_ENABLED var is set
.packet_periodic:
only:
variables:
- $PERIODIC_CI_ENABLED
allow_failure: true
extends: .packet
# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
packet_ubuntu20-calico-all-in-one:
stage: deploy-part1
extends: .packet_pr_short
variables:
RESET_CHECK: "true"
# ### PR JOBS PART2
packet_ubuntu20-crio:
extends: .packet_pr_manual
packet_ubuntu22-calico-all-in-one:
extends: .packet_pr
packet_ubuntu22-calico-all-in-one-upgrade:
extends: .packet_pr
variables:
UPGRADE_TEST: graceful
packet_ubuntu24-calico-etcd-datastore:
extends: .packet_pr
packet_almalinux9-crio:
extends: .packet_pr
packet_almalinux9-kube-ovn:
extends: .packet_pr
packet_debian11-calico-collection:
extends: .packet_pr
packet_debian11-macvlan:
extends: .packet_pr
packet_debian12-cilium:
extends: .packet_pr
packet_almalinux8-calico:
extends: .packet_pr
packet_rockylinux8-calico:
extends: .packet_pr
packet_rockylinux9-cilium:
extends: .packet_pr
variables:
RESET_CHECK: "true"
# Need an update of the container image to use schema v2
# update: quay.io/kubespray/vm-amazon-linux-2:latest
packet_amazon-linux-2-all-in-one:
extends: .packet_pr_manual
rules:
- when: manual
allow_failure: true
packet_opensuse15-6-calico:
extends: .packet_pr
packet_ubuntu20-cilium-sep:
extends: .packet_pr
packet_openeuler24-calico:
extends: .packet_pr
packet_ubuntu20-calico-all-in-one-hardening:
extends: .packet_pr
## Extended
packet_debian11-docker:
extends: .packet_pr_extended
packet_debian12-docker:
extends: .packet_pr_extended
packet_debian12-calico:
extends: .packet_pr_extended
packet_almalinux9-calico-remove-node:
extends: .packet_pr_extended
variables:
REMOVE_NODE_CHECK: "true"
REMOVE_NODE_NAME: "instance-3"
packet_rockylinux9-calico:
extends: .packet_pr_extended
packet_almalinux9-calico:
extends: .packet_pr_extended
packet_almalinux9-docker:
extends: .packet_pr_extended
packet_opensuse15-6-docker-cilium:
extends: .packet_pr_extended
packet_ubuntu24-calico-all-in-one:
extends: .packet_pr_extended
packet_ubuntu20-calico-etcd-kubeadm:
extends: .packet_pr_extended
packet_ubuntu24-all-in-one-docker:
extends: .packet_pr_extended
packet_ubuntu22-all-in-one-docker:
extends: .packet_pr_extended
# ### MANUAL JOBS
packet_fedora39-crio:
extends: .packet_pr_manual
packet_ubuntu20-flannel-ha:
extends: .packet_pr_manual
packet_ubuntu20-all-in-one-docker:
extends: .packet_pr_manual
packet_ubuntu20-flannel-ha-once:
extends: .packet_pr_manual
packet_fedora39-calico-swap-selinux:
extends: .packet_pr_manual
packet_almalinux9-calico-ha-ebpf:
extends: .packet_pr_manual
packet_almalinux9-calico-nodelocaldns-secondary:
extends: .packet_pr_manual
packet_debian11-custom-cni:
extends: .packet_pr_manual
packet_debian11-kubelet-csr-approver:
extends: .packet_pr_manual
packet_debian12-custom-cni-helm:
extends: .packet_pr_manual
packet_ubuntu20-calico-ha-wireguard:
extends: .packet_pr_manual
# PERIODIC
packet_fedora40-docker-calico:
stage: deploy-extended
extends: .packet_periodic
variables:
RESET_CHECK: "true"
packet_fedora39-calico-selinux:
stage: deploy-extended
extends: .packet_periodic
packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
stage: deploy-extended
extends: .packet_periodic
variables:
UPGRADE_TEST: basic
packet_debian11-calico-upgrade-once:
stage: deploy-extended
extends: .packet_periodic
variables:
UPGRADE_TEST: graceful
packet_ubuntu20-calico-ha-recover:
stage: deploy-extended
extends: .packet_periodic
variables:
RECOVER_CONTROL_PLANE_TEST: "true"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
packet_ubuntu20-calico-ha-recover-noquorum:
stage: deploy-extended
extends: .packet_periodic
variables:
RECOVER_CONTROL_PLANE_TEST: "true"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
packet_debian11-calico-upgrade:
stage: deploy-extended
extends: .packet_periodic
variables:
UPGRADE_TEST: graceful
packet_debian12-cilium-svc-proxy:
stage: deploy-extended
extends: .packet_periodic
-239
View File
@@ -1,239 +0,0 @@
---
# Tests for contrib/terraform/
.terraform_install:
extends: .job
needs:
- ci-not-authorized
- pipeline-image
stage: deploy-part1
before_script:
- update-alternatives --install /usr/bin/python python /usr/bin/python3 1
- ./tests/scripts/rebase.sh
- ./tests/scripts/testcases_prepare.sh
- ./tests/scripts/terraform_install.sh
# Set Ansible config
- cp ansible.cfg ~/.ansible.cfg
# Prepare inventory
- cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
- ln -s contrib/terraform/$PROVIDER/hosts
- terraform -chdir="contrib/terraform/$PROVIDER" init
# Copy SSH keypair
- mkdir -p ~/.ssh
- echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
- chmod 400 ~/.ssh/id_rsa
- echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
- mkdir -p contrib/terraform/$PROVIDER/group_vars
# Random subnet to avoid routing conflicts
- export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
.terraform_validate:
extends: .terraform_install
tags: [ffci]
only: ['master', /^pr-.*$/]
script:
- terraform -chdir="contrib/terraform/$PROVIDER" validate
- terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
stage: test
needs:
- pipeline-image
.terraform_apply:
extends: .terraform_install
tags: [ffci]
stage: deploy-extended
when: manual
only: [/^pr-.*$/]
artifacts:
when: always
paths:
- cluster-dump/
variables:
ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
ANSIBLE_INVENTORY: hosts
CI_PLATFORM: tf
TF_VAR_ssh_user: $SSH_USER
TF_VAR_cluster_name: $CI_JOB_ID
script:
- tests/scripts/testcases_run.sh
after_script:
# Cleanup regardless of exit code
- ./tests/scripts/testcases_cleanup.sh
tf-validate-openstack:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: openstack
CLUSTER: $CI_COMMIT_REF_NAME
tf-validate-equinix:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: equinix
CLUSTER: $CI_COMMIT_REF_NAME
tf-validate-aws:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: aws
CLUSTER: $CI_COMMIT_REF_NAME
tf-validate-exoscale:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: exoscale
tf-validate-hetzner:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: hetzner
tf-validate-vsphere:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: vsphere
CLUSTER: $CI_COMMIT_REF_NAME
tf-validate-upcloud:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: upcloud
CLUSTER: $CI_COMMIT_REF_NAME
tf-validate-nifcloud:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: nifcloud
# tf-packet-ubuntu20-default:
# extends: .terraform_apply
# variables:
# TF_VERSION: $TERRAFORM_VERSION
# PROVIDER: packet
# CLUSTER: $CI_COMMIT_REF_NAME
# TF_VAR_number_of_k8s_masters: "1"
# TF_VAR_number_of_k8s_nodes: "1"
# TF_VAR_plan_k8s_masters: t1.small.x86
# TF_VAR_plan_k8s_nodes: t1.small.x86
# TF_VAR_metro: am
# TF_VAR_public_key_path: ""
# TF_VAR_operating_system: ubuntu_20_04
.ovh_variables: &ovh_variables
OS_AUTH_URL: https://auth.cloud.ovh.net/v3
OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
OS_PROJECT_NAME: "9361447987648822"
OS_USER_DOMAIN_NAME: Default
OS_PROJECT_DOMAIN_ID: default
OS_USERNAME: 8XuhBMfkKVrk
OS_REGION_NAME: UK1
OS_INTERFACE: public
OS_IDENTITY_API_VERSION: "3"
# Elastx is generously donating resources for Kubespray on Openstack CI
# Contacts: @gix @bl0m1
.elastx_variables: &elastx_variables
OS_AUTH_URL: https://ops.elastx.cloud:5000
OS_PROJECT_ID: 564c6b461c6b44b1bb19cdb9c2d928e4
OS_PROJECT_NAME: kubespray_ci
OS_USER_DOMAIN_NAME: Default
OS_PROJECT_DOMAIN_ID: default
OS_USERNAME: kubespray@root314.com
OS_REGION_NAME: se-sto
OS_INTERFACE: public
OS_IDENTITY_API_VERSION: "3"
TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
tf-elastx_cleanup:
tags: [ffci]
image: python
variables:
<<: *elastx_variables
before_script:
- pip install -r scripts/openstack-cleanup/requirements.txt
script:
- ./scripts/openstack-cleanup/main.py
allow_failure: true
tf-elastx_ubuntu20-calico:
extends: .terraform_apply
stage: deploy-part1
when: on_success
allow_failure: true
variables:
<<: *elastx_variables
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: openstack
CLUSTER: $CI_COMMIT_REF_NAME
ANSIBLE_TIMEOUT: "60"
SSH_USER: ubuntu
TF_VAR_number_of_k8s_masters: "1"
TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
TF_VAR_number_of_etcd: "0"
TF_VAR_number_of_k8s_nodes: "1"
TF_VAR_number_of_k8s_nodes_no_floating_ip: "0"
TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
TF_VAR_number_of_bastions: "0"
TF_VAR_number_of_k8s_masters_no_etcd: "0"
TF_VAR_floatingip_pool: "elx-public1"
TF_VAR_dns_nameservers: '["1.1.1.1", "8.8.8.8", "8.8.4.4"]'
TF_VAR_use_access_ip: "0"
TF_VAR_external_net: "600b8501-78cb-4155-9c9f-23dfcba88828"
TF_VAR_network_name: "ci-$CI_JOB_ID"
TF_VAR_az_list: '["sto1"]'
TF_VAR_az_list_node: '["sto1"]'
TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b # v1-standard-2
TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b # v1-standard-2
TF_VAR_image: ubuntu-20.04-server-latest
TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
# OVH voucher expired, commenting job until things are sorted out
# tf-ovh_cleanup:
# stage: unit-tests
# tags: [light]
# image: python
# environment: ovh
# variables:
# <<: *ovh_variables
# before_script:
# - pip install -r scripts/openstack-cleanup/requirements.txt
# script:
# - ./scripts/openstack-cleanup/main.py
# tf-ovh_ubuntu20-calico:
# extends: .terraform_apply
# when: on_success
# environment: ovh
# variables:
# <<: *ovh_variables
# TF_VERSION: $TERRAFORM_VERSION
# PROVIDER: openstack
# CLUSTER: $CI_COMMIT_REF_NAME
# ANSIBLE_TIMEOUT: "60"
# SSH_USER: ubuntu
# TF_VAR_number_of_k8s_masters: "0"
# TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
# TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
# TF_VAR_number_of_etcd: "0"
# TF_VAR_number_of_k8s_nodes: "0"
# TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
# TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
# TF_VAR_number_of_bastions: "0"
# TF_VAR_number_of_k8s_masters_no_etcd: "0"
# TF_VAR_use_neutron: "0"
# TF_VAR_floatingip_pool: "Ext-Net"
# TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
# TF_VAR_network_name: "Ext-Net"
# TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229" # s1-8
# TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229" # s1-8
# TF_VAR_image: "Ubuntu 20.04"
# TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
-81
View File
@@ -1,81 +0,0 @@
---
.vagrant:
extends: .testcases
needs:
- ci-not-authorized
variables:
CI_PLATFORM: "vagrant"
SSH_USER: "vagrant"
VAGRANT_DEFAULT_PROVIDER: "libvirt"
KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
DOCKER_NAME: vagrant
VAGRANT_ANSIBLE_TAGS: facts
VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
tags: [ffci-vm-large]
# only: [/^pr-.*$/]
# except: ['triggers']
image: quay.io/kubespray/vm-kubespray-ci:v13
services: []
before_script:
- echo $USER
- python3 -m venv citest
- source citest/bin/activate
- vagrant plugin expunge --reinstall --force --no-tty
- vagrant plugin install vagrant-libvirt
- pip install --no-compile --no-cache-dir pip -U
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
- ./tests/scripts/vagrant_clean.sh
script:
- ./tests/scripts/testcases_run.sh
cache:
key: $CI_JOB_NAME_SLUG
paths:
- .vagrant.d/boxes
- .cache/pip
policy: pull-push # TODO: change to "pull" when not on main
vagrant_ubuntu24-calico-dual-stack:
stage: deploy-extended
extends: .vagrant
rules:
- if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
when: on_success
allow_failure: false
vagrant_ubuntu24-calico-ipv6only-stack:
stage: deploy-extended
extends: .vagrant
rules:
- if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
when: on_success
allow_failure: false
vagrant_ubuntu20-flannel:
stage: deploy-part1
extends: .vagrant
when: on_success
allow_failure: false
vagrant_ubuntu20-flannel-collection:
stage: deploy-extended
extends: .vagrant
when: manual
vagrant_ubuntu20-kube-router-sep:
stage: deploy-extended
extends: .vagrant
when: manual
# Service proxy test fails connectivity testing
vagrant_ubuntu20-kube-router-svc-proxy:
stage: deploy-extended
extends: .vagrant
when: manual
vagrant_fedora39-kube-router:
stage: deploy-extended
extends: .vagrant
when: manual
# FIXME: this test if broken (perma-failing)
View File
-4
View File
@@ -1,4 +0,0 @@
all
exclude_rule 'MD013'
exclude_rule 'MD029'
rule 'MD007', :indent => 2
-1
View File
@@ -1 +0,0 @@
style "#{File.dirname(__FILE__)}/.md_style.rb"
View File
-110
View File
@@ -1,110 +0,0 @@
---
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: check-added-large-files
- id: check-case-conflict
- id: check-executables-have-shebangs
- id: check-xml
- id: check-merge-conflict
- id: detect-private-key
- id: end-of-file-fixer
- id: forbid-new-submodules
- id: requirements-txt-fixer
- id: trailing-whitespace
- repo: https://github.com/adrienverge/yamllint.git
rev: v1.35.1
hooks:
- id: yamllint
args: [--strict]
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.10.0.1
hooks:
- id: shellcheck
args: ["--severity=error"]
exclude: "^.git"
files: "\\.sh$"
- repo: https://github.com/ansible/ansible-lint
rev: v25.1.1
hooks:
- id: ansible-lint
additional_dependencies:
- jmespath==1.0.1
- netaddr==1.3.0
- distlib
- repo: https://github.com/golangci/misspell
rev: v0.6.0
hooks:
- id: misspell
exclude: "OWNERS_ALIASES$"
- repo: local
hooks:
- id: collection-build-install
name: Build and install kubernetes-sigs.kubespray Ansible collection
language: python
additional_dependencies:
- ansible-core>=2.16.4
- distlib
entry: tests/scripts/collection-build-install.sh
pass_filenames: false
- id: generate-docs-sidebar
name: generate-docs-sidebar
entry: scripts/gen_docs_sidebar.sh
language: script
pass_filenames: false
- id: ci-matrix
name: ci-matrix
entry: tests/scripts/md-table/main.py
language: python
pass_filenames: false
additional_dependencies:
- jinja2
- pathlib
- pyaml
- id: check-galaxy-version
name: Verify correct version for galaxy.yml
entry: scripts/galaxy_version.py
language: python
pass_filenames: false
additional_dependencies:
- ruamel.yaml
- id: jinja-syntax-check
name: jinja-syntax-check
entry: tests/scripts/check-templates.py
language: python
types:
- jinja
additional_dependencies:
- jinja2
- id: propagate-ansible-variables
name: Update static files referencing default kubespray values
language: python
additional_dependencies:
- ansible-core>=2.16.4
entry: scripts/propagate_ansible_variables.yml
pass_filenames: false
- id: check-checksums-sorted
name: Check that our checksums are correctly sorted by version
entry: scripts/assert-sorted-checksums.yml
language: python
pass_filenames: false
additional_dependencies:
- ansible
- repo: https://github.com/markdownlint/markdownlint
rev: v0.12.0
hooks:
- id: markdownlint
exclude: "^.github|(^docs/_sidebar\\.md$)"
-28
View File
@@ -1,28 +0,0 @@
---
extends: default
ignore: |
.git/
.github/
# Generated file
tests/files/custom_cni/cilium.yaml
# https://ansible.readthedocs.io/projects/lint/rules/yaml/
rules:
braces:
min-spaces-inside: 0
max-spaces-inside: 1
brackets:
min-spaces-inside: 0
max-spaces-inside: 1
comments:
min-spaces-from-content: 1
# https://github.com/adrienverge/yamllint/issues/384
comments-indentation: false
indentation:
spaces: 2
indent-sequences: consistent
line-length: disable
new-line-at-end-of-file: disable
octal-values:
forbid-implicit-octal: true # yamllint defaults to false
forbid-explicit-octal: true # yamllint defaults to false
-1
View File
@@ -1 +0,0 @@
# See our release notes on [GitHub](https://github.com/kubernetes-sigs/kubespray/releases)
-1
View File
@@ -1 +0,0 @@
kubespray.io
-47
View File
@@ -1,47 +0,0 @@
# Contributing guidelines
## How to become a contributor and submit your own code
### Environment setup
It is recommended to use filter to manage the GitHub email notification, see [examples for setting filters to Kubernetes Github notifications](https://github.com/kubernetes/community/blob/master/communication/best-practices.md#examples-for-setting-filters-to-kubernetes-github-notifications)
To install development dependencies you can set up a python virtual env with the necessary dependencies:
```ShellSession
virtualenv venv
source venv/bin/activate
pip install -r tests/requirements.txt
ansible-galaxy install -r tests/requirements.yml
```
#### Linting
Kubespray uses [pre-commit](https://pre-commit.com) hook configuration to run several linters, please install this tool and use it to run validation tests before submitting a PR.
```ShellSession
pre-commit install
pre-commit run -a # To run pre-commit hook on all files in the repository, even if they were not modified
```
#### Molecule
[molecule](https://github.com/ansible-community/molecule) is designed to help the development and testing of Ansible roles. In Kubespray you can run it all for all roles with `./tests/scripts/molecule_run.sh` or for a specific role (that you are working with) with `molecule test` from the role directory (`cd roles/my-role`).
When developing or debugging a role it can be useful to run `molecule create` and `molecule converge` separately. Then you can use `molecule login` to SSH into the test environment.
#### Vagrant
Vagrant with VirtualBox or libvirt driver helps you to quickly spin test clusters to test things end to end. See [README.md#vagrant](README.md)
### Contributing A Patch
1. Submit an issue describing your proposed change to the repo in question.
2. The [repo owners](OWNERS) will respond to your issue promptly.
3. Fork the desired repo, develop and test your code changes.
4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
5. Addess any pre-commit validation failures.
6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
7. Submit a pull request.
8. Work with the reviewers on their suggestions.
9. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits (<https://blog.carbonfive.com/always-squash-and-rebase-your-git-commits/>) before final merger of your contribution.
-50
View File
@@ -1,50 +0,0 @@
# syntax=docker/dockerfile:1
# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
# Some tools like yamllint need this
# Pip needs this as well at the moment to install ansible
# (and potentially other packages)
# See: https://github.com/pypa/pip/issues/10219
ENV LANG=C.UTF-8 \
DEBIAN_FRONTEND=noninteractive \
PYTHONDONTWRITEBYTECODE=1
WORKDIR /kubespray
# hadolint ignore=DL3008
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
apt-get update -q \
&& apt-get install -yq --no-install-recommends \
curl \
python3 \
python3-pip \
sshpass \
vim \
rsync \
openssh-client \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /var/log/*
RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
--mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \
pip install --no-compile --no-cache-dir -r requirements.txt \
&& find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \;
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
&& curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
&& echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
&& chmod a+x /usr/local/bin/kubectl
COPY *.yml ./
COPY *.cfg ./
COPY roles ./roles
COPY contrib ./contrib
COPY inventory ./inventory
COPY library ./library
COPY extra_playbooks ./extra_playbooks
COPY playbooks ./playbooks
COPY plugins ./plugins
-201
View File
@@ -1,201 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2016 Kubespray
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-8
View File
@@ -1,8 +0,0 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- kubespray-approvers
reviewers:
- kubespray-reviewers
emeritus_approvers:
- kubespray-emeritus_approvers
-26
View File
@@ -1,26 +0,0 @@
aliases:
kubespray-approvers:
- cristicalin
- floryut
- liupeng0518
- mzaian
- oomichi
- yankay
- ant31
- vannten
kubespray-reviewers:
- cyclinder
- erikjiang
- mrfreezeex
- mzaian
- tico88612
- vannten
- yankay
kubespray-emeritus_approvers:
- atoms
- chadswen
- luckysb
- mattymo
- miouge1
- riverzhang
- woopstar
+23 -224
View File
@@ -1,234 +1,33 @@
# Deploy a Production Ready Kubernetes Cluster
vagrant-k8s
===========
Scripts to create libvirt lab with vagrant and prepare some stuff for `k8s` deployment with `kargo`.
![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-sigs/kubespray/master/docs/img/kubernetes-logo.png)
If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
You can get your invite [here](http://slack.k8s.io/)
Requirements
============
- Can be deployed on **[AWS](docs/cloud_providers/aws.md), GCE, [Azure](docs/cloud_providers/azure.md), [OpenStack](docs/cloud_controllers/openstack.md), [vSphere](docs/cloud_controllers/vsphere.md), [Equinix Metal](docs/cloud_providers/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
- **Highly available** cluster
- **Composable** (Choice of the network plugin for instance)
- Supports most popular **Linux distributions**
- **Continuous integration tests**
* `libvirt`
* `vagrant`
* `vagrant-libvirt` plugin
* `$USER` should be able to connect to libvirt (test with `virsh list --all`)
## Quick Start
How-to
======
Below are several ways to use Kubespray to deploy a Kubernetes cluster.
* Prepare the virtual lab:
### Docker
Ensure you have installed Docker then
```ShellSession
docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
--mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
quay.io/kubespray/kubespray:v2.27.0 bash
# Inside the container you may now run the kubespray playbooks:
ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
```
### Ansible
#### Usage
See [Getting started](/docs/getting_started/getting-started.md)
#### Collection
See [here](docs/ansible/ansible_collection.md) if you wish to use this repository as an Ansible collection
### Vagrant
For Vagrant we need to install Python dependencies for provisioning tasks.
Check that ``Python`` and ``pip`` are installed:
```ShellSession
python -V && pip -V
```
If this returns the version of the software, you're good to go. If not, download and install Python from here <https://www.python.org/downloads/source/>
Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
then run the following step:
```ShellSession
```bash
export VAGRANT_POOL="10.100.0.0/16"
git clone https://github.com/adidenko/vagrant-k8s
cd vagrant-k8s
vagrant up
```
## Documents
* Login to master node and deploy k8s with kargo:
- [Requirements](#requirements)
- [Kubespray vs ...](docs/getting_started/comparisons.md)
- [Getting started](docs/getting_started/getting-started.md)
- [Setting up your first cluster](docs/getting_started/setting-up-your-first-cluster.md)
- [Ansible inventory and tags](docs/ansible/ansible.md)
- [Integration with existing ansible repo](docs/operations/integration.md)
- [Deployment data variables](docs/ansible/vars.md)
- [DNS stack](docs/advanced/dns-stack.md)
- [HA mode](docs/operations/ha-mode.md)
- [Network plugins](#network-plugins)
- [Vagrant install](docs/developers/vagrant.md)
- [Flatcar Container Linux bootstrap](docs/operating_systems/flatcar.md)
- [Fedora CoreOS bootstrap](docs/operating_systems/fcos.md)
- [openSUSE setup](docs/operating_systems/opensuse.md)
- [Downloaded artifacts](docs/advanced/downloads.md)
- [Equinix Metal](docs/cloud_providers/equinix-metal.md)
- [OpenStack](docs/cloud_controllers/openstack.md)
- [vSphere](docs/cloud_controllers/vsphere.md)
- [Large deployments](docs/operations/large-deployments.md)
- [Adding/replacing a node](docs/operations/nodes.md)
- [Upgrades basics](docs/operations/upgrades.md)
- [Air-Gap installation](docs/operations/offline-environment.md)
- [NTP](docs/advanced/ntp.md)
- [Hardening](docs/operations/hardening.md)
- [Mirror](docs/operations/mirror.md)
- [Roadmap](docs/roadmap/roadmap.md)
## Supported Linux Distributions
- **Flatcar Container Linux by Kinvolk**
- **Debian** Bookworm, Bullseye
- **Ubuntu** 20.04, 22.04, 24.04
- **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
- **Fedora** 39, 40
- **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
- **openSUSE** Leap 15.x/Tumbleweed
- **Oracle Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
- **Alma Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
- **Rocky Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
- **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
- **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md))
Note:
- Upstart/SysV init based OS types are not supported.
- [Kernel requirements](docs/operations/kernel-requirements.md) (please read if the OS kernel version is < 4.19).
## Supported Components
<!-- BEGIN ANSIBLE MANAGED BLOCK -->
- Core
- [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
- [etcd](https://github.com/etcd-io/etcd) 3.5.16
- [docker](https://www.docker.com/) 28.0
- [containerd](https://containerd.io/) 2.0.4
- [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
- Network Plugin
- [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
- [calico](https://github.com/projectcalico/calico) 3.29.3
- [cilium](https://github.com/cilium/cilium) 1.15.9
- [flannel](https://github.com/flannel-io/flannel) 0.22.0
- [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
- [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
- [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
- [weave](https://github.com/rajch/weave) 2.8.7
- [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
- Application
- [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
- [coredns](https://github.com/coredns/coredns) 1.11.3
- [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.1
- [argocd](https://argoproj.github.io/) 2.14.5
- [helm](https://helm.sh/) 3.16.4
- [metallb](https://metallb.universe.tf/) 0.13.9
- [registry](https://github.com/distribution/distribution) 2.8.1
- Storage Plugin
- [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
- [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
- [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
- [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
- [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
- [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
- [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
- [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
- [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
<!-- END ANSIBLE MANAGED BLOCK -->
## Container Runtime Notes
- The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)
## Requirements
- **Minimum required version of Kubernetes is v1.30**
- **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**
- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md))
- The target servers are configured to allow **IPv4 forwarding**.
- If using IPv6 for pods and services, the target servers are configured to allow **IPv6 forwarding**.
- The **firewalls are not managed**, you'll need to implement your own rules the way you used to.
in order to avoid any issue during deployment you should disable your firewall.
- If kubespray is run from non-root user account, correct privilege escalation method
should be configured in the target servers. Then the `ansible_become` flag
or command parameters `--become or -b` should be specified.
Hardware:
These limits are safeguarded by Kubespray. Actual requirements for your workload can differ. For a sizing guide go to the [Building Large Clusters](https://kubernetes.io/docs/setup/cluster-large/#size-of-master-and-master-components) guide.
- Control Plane
- Memory: 2 GB
- Worker Node
- Memory: 1 GB
## Network Plugins
You can choose among ten network plugins. (default: `calico`, except Vagrant uses `flannel`)
- [flannel](docs/CNI/flannel.md): gre/vxlan (layer 2) networking.
- [Calico](https://docs.tigera.io/calico/latest/about/) is a networking and network policy provider. Calico supports a flexible set of networking options
designed to give you the most efficient networking across a range of situations, including non-overlay
and overlay networks, with or without BGP. Calico uses the same engine to enforce network policy for hosts,
pods, and (if using Istio and Envoy) applications at the service mesh layer.
- [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
(Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
- [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
- [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational
simplicity and high performance: it uses IPVS to provide Kube Services Proxy (if setup to replace kube-proxy),
iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers).
It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs.
- [macvlan](docs/CNI/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.
- [multus](docs/CNI/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
- [custom_cni](roles/network-plugin/custom_cni/) : You can specify some manifests that will be applied to the clusters to bring you own CNI and use non-supported ones by Kubespray.
See `tests/files/custom_cni/README.md` and `tests/files/custom_cni/values.yaml`for an example with a CNI provided by a Helm Chart.
The network plugin to use is defined by the variable `kube_network_plugin`. There is also an
option to leverage built-in cloud provider networking instead.
See also [Network checker](docs/advanced/netcheck.md).
## Ingress Plugins
- [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller.
- [metallb](docs/ingress/metallb.md): the MetalLB bare-metal service LoadBalancer provider.
## Community docs and resources
- [kubernetes.io/docs/setup/production-environment/tools/kubespray/](https://kubernetes.io/docs/setup/production-environment/tools/kubespray/)
- [kubespray, monitoring and logging](https://github.com/gregbkr/kubernetes-kargo-logging-monitoring) by @gregbkr
- [Deploy Kubernetes w/ Ansible & Terraform](https://rsmitty.github.io/Terraform-Ansible-Kubernetes/) by @rsmitty
- [Deploy a Kubernetes Cluster with Kubespray (video)](https://www.youtube.com/watch?v=CJ5G4GpqDy0)
## Tools and projects on top of Kubespray
- [Digital Rebar Provision](https://github.com/digitalrebar/provision/blob/v4/doc/integrations/ansible.rst)
- [Terraform Contrib](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/terraform)
- [Kubean](https://github.com/kubean-io/kubean)
## CI Tests
[![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/-/pipelines)
CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
See the [test matrix](docs/developers/test_cases.md) for details.
```bash
vagrant ssh $USER-k8s-01
# Inside your master VM run this:
sudo su -
./deploy-k8s.kargo.sh
```
-85
View File
@@ -1,85 +0,0 @@
# Release Process
The Kubespray Project is released on an as-needed basis. The process is as follows:
1. An issue is proposing a new release with a changelog since the last release. Please see [a good sample issue](https://github.com/kubernetes-sigs/kubespray/issues/8325)
1. At least one of the [approvers](OWNERS_ALIASES) must approve this release
1. (Only for major releases) The `kube_version_min_required` variable is set to `n-1`
1. (Only for major releases) Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables.
1. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details.
1. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes
1. (Only for major releases) An approver creates a release branch in the form `release-X.Y`
1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request.
1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request.
1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.
1. The release issue is closed
1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
1. Create/Update Issue for upgradeing kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)
## Major/minor releases and milestones
* For major releases (vX.Y) Kubespray maintains one branch (`release-X.Y`). Minor releases (vX.Y.Z) are available only as tags.
* Security patches and bugs might be backported.
* Fixes for major releases (vX.Y) and minor releases (vX.Y.Z) are delivered
via maintenance releases (vX.Y.Z) and assigned to the corresponding open
[GitHub milestone](https://github.com/kubernetes-sigs/kubespray/milestones).
That milestone remains open for the major/minor releases support lifetime,
which ends once the milestone is closed. Then only a next major or minor release
can be done.
* Kubespray major and minor releases are bound to the given `kube_version` major/minor
version numbers and other components' arbitrary versions, like etcd or network plugins.
Older or newer component versions are not supported and not tested for the given
release (even if included in the checksum variables, like `kubeadm_checksums`).
* There is no unstable releases and no APIs, thus Kubespray doesn't follow
[semver](https://semver.org/). Every version describes only a stable release.
Breaking changes, if any introduced by changed defaults or non-contrib ansible roles'
playbooks, shall be described in the release notes. Other breaking changes, if any in
the contributed addons or bound versions of Kubernetes and other components, are
considered out of Kubespray scope and are up to the components' teams to deal with and
document.
* Minor releases can change components' versions, but not the major `kube_version`.
Greater `kube_version` requires a new major or minor release. For example, if Kubespray v2.0.0
is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: 3.0.6`,
then Kubespray v2.1.0 may be bound to only minor changes to `kube_version`, like v1.5.1
and *any* changes to other components, like etcd v4, or calico 1.2.3.
And Kubespray v3.x.x shall be bound to `kube_version: 2.x.x` respectively.
## Release note creation
You can create a release note with:
```shell
export GITHUB_TOKEN=<your-github-token>
export ORG=kubernetes-sigs
export REPO=kubespray
release-notes --start-sha <The start commit-id> --end-sha <The end commit-id> --dependencies=false --output=/tmp/kubespray-release-note --required-author=""
```
If the release note file(/tmp/kubespray-release-note) contains "### Uncategorized" pull requests, those pull requests don't have a valid kind label(`kind/feature`, etc.).
It is necessary to put a valid label on each pull request and run the above release-notes command again to get a better release note
## Container image creation
The container image `quay.io/kubespray/kubespray:vX.Y.Z` can be created from Dockerfile of the kubespray root directory:
```shell
cd kubespray/
nerdctl build -t quay.io/kubespray/kubespray:vX.Y.Z .
nerdctl push quay.io/kubespray/kubespray:vX.Y.Z
```
The container image `quay.io/kubespray/vagrant:vX.Y.Z` can be created from build.sh of test-infra/vagrant-docker/:
```shell
cd kubespray/test-infra/vagrant-docker/
./build vX.Y.Z
```
Please note that the above operation requires the permission to push container images into quay.io/kubespray/.
If you don't have the permission, please ask it on the #kubespray-dev channel.
-15
View File
@@ -1,15 +0,0 @@
# Defined below are the security contacts for this repo.
#
# They are the contact point for the Product Security Committee to reach out
# to for triaging and handling of incoming issues.
#
# The below names agree to abide by the
# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy)
# and will be removed and replaced if they violate that agreement.
#
# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
# INSTRUCTIONS AT https://kubernetes.io/security/
floryut
ant31
VannTen
yankay
Vendored
+80 -297
View File
@@ -1,318 +1,101 @@
# -*- mode: ruby -*-
# # vi: set ft=ruby :
# vi: set ft=ruby :
# For help on using kubespray with vagrant, check out docs/developers/vagrant.md
ENV["VAGRANT_DEFAULT_PROVIDER"] = "libvirt"
pool = ENV["VAGRANT_POOL"] || "10.210.0.0/16"
prefix = pool.gsub(/\.\d+\.\d+\/16$/, "")
require 'fileutils'
$num_instances = 7
$vm_memory = 2048
$vm_cpus = 2
Vagrant.require_version ">= 2.0.0"
$user = ENV["USER"]
$public_subnet = prefix.to_s + ".0"
$private_subnet = prefix.to_s + ".1"
$mgmt_cidr = prefix.to_s + ".2.0/24"
CONFIG = File.join(File.dirname(__FILE__), ENV['KUBESPRAY_VAGRANT_CONFIG'] || 'vagrant/config.rb')
$instance_name_prefix = "#{$user}-k8s"
FLATCAR_URL_TEMPLATE = "https://%s.release.flatcar-linux.net/amd64-usr/current/flatcar_production_vagrant.json"
# Boxes with libvirt provider support:
#$box = "yk0/ubuntu-xenial" #900M
#$box = "centos/7"
$box = "nrclark/xenial64-minimal-libvirt"
# Uniq disk UUID for libvirt
DISK_UUID = Time.now.utc.to_i
# Create SSH keys for future lab
system 'bash ssh-keygen.sh'
SUPPORTED_OS = {
"flatcar-stable" => {box: "flatcar-stable", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["stable"]},
"flatcar-beta" => {box: "flatcar-beta", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["beta"]},
"flatcar-alpha" => {box: "flatcar-alpha", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["alpha"]},
"flatcar-edge" => {box: "flatcar-edge", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["edge"]},
"ubuntu2004" => {box: "generic/ubuntu2004", user: "vagrant"},
"ubuntu2204" => {box: "generic/ubuntu2204", user: "vagrant"},
"ubuntu2404" => {box: "bento/ubuntu-24.04", user: "vagrant"},
"centos8" => {box: "centos/8", user: "vagrant"},
"centos8-bento" => {box: "bento/centos-8", user: "vagrant"},
"almalinux8" => {box: "almalinux/8", user: "vagrant"},
"almalinux8-bento" => {box: "bento/almalinux-8", user: "vagrant"},
"almalinux9" => {box: "almalinux/9", user: "vagrant"},
"rockylinux8" => {box: "rockylinux/8", user: "vagrant"},
"rockylinux9" => {box: "rockylinux/9", user: "vagrant"},
"fedora39" => {box: "fedora/39-cloud-base", user: "vagrant"},
"fedora40" => {box: "fedora/40-cloud-base", user: "vagrant"},
"fedora39-arm64" => {box: "bento/fedora-39-arm64", user: "vagrant"},
"fedora40-arm64" => {box: "bento/fedora-40", user: "vagrant"},
"opensuse" => {box: "opensuse/Leap-15.6.x86_64", user: "vagrant"},
"opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
"oraclelinux" => {box: "generic/oracle7", user: "vagrant"},
"oraclelinux8" => {box: "generic/oracle8", user: "vagrant"},
"rhel8" => {box: "generic/rhel8", user: "vagrant"},
"debian11" => {box: "debian/bullseye64", user: "vagrant"},
"debian12" => {box: "debian/bookworm64", user: "vagrant"},
}
if File.exist?(CONFIG)
require CONFIG
end
# Defaults for config options defined in CONFIG
$num_instances ||= 3
$instance_name_prefix ||= "k8s"
$vm_gui ||= false
$vm_memory ||= 2048
$vm_cpus ||= 2
$shared_folders ||= {}
$forwarded_ports ||= {}
$subnet ||= "172.18.8"
$subnet_ipv6 ||= "fd3c:b398:0698:0756"
$os ||= "ubuntu2004"
$network_plugin ||= "flannel"
$inventories ||= []
# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni
$multi_networking ||= "False"
$download_run_once ||= "True"
$download_force_cache ||= "False"
# Modify those to have separate groups (for instance, to test separate etcd:)
# first_control_plane = 1
# first_etcd = 4
# control_plane_instances = 3
# etcd_instances = 3
$first_node ||= 1
$first_control_plane ||= 1
$first_etcd ||= 1
# The first three nodes are etcd servers
$etcd_instances ||= [$num_instances, 3].min
# The first two nodes are kube masters
$control_plane_instances ||= [$num_instances, 2].min
# All nodes are kube nodes
$kube_node_instances ||= $num_instances - $first_node + 1
# The following only works when using the libvirt provider
$kube_node_instances_with_disks ||= false
$kube_node_instances_with_disks_size ||= "20G"
$kube_node_instances_with_disks_number ||= 2
$override_disk_size ||= false
$disk_size ||= "20GB"
$local_path_provisioner_enabled ||= "False"
$local_path_provisioner_claim_root ||= "/opt/local-path-provisioner/"
$libvirt_nested ||= false
# boolean or string (e.g. "-vvv")
$ansible_verbosity ||= false
$ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || ""
$vagrant_dir ||= File.join(File.dirname(__FILE__), ".vagrant")
$playbook ||= "cluster.yml"
$extra_vars ||= {}
host_vars = {}
# throw error if os is not supported
if ! SUPPORTED_OS.key?($os)
puts "Unsupported OS: #{$os}"
puts "Supported OS are: #{SUPPORTED_OS.keys.join(', ')}"
exit 1
end
$box = SUPPORTED_OS[$os][:box]
if Vagrant.has_plugin?("vagrant-proxyconf")
$no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
(1..$num_instances).each do |i|
$no_proxy += ",#{$subnet}.#{i+100}"
end
# Create nodes list for future kargo deployment
nodes=""
(2..$num_instances).each do |i|
ip = "#{$private_subnet}.#{i+10}"
nodes = "#{nodes}#{ip}\n"
end
File.open("nodes", 'w') { |file| file.write(nodes) }
# Create the lab
Vagrant.configure("2") do |config|
config.vm.box = $box
if SUPPORTED_OS[$os].has_key? :box_url
config.vm.box_url = SUPPORTED_OS[$os][:box_url]
end
config.ssh.username = SUPPORTED_OS[$os][:user]
# plugin conflict
if Vagrant.has_plugin?("vagrant-vbguest") then
config.vbguest.auto_update = false
end
# always use Vagrants insecure key
config.ssh.insert_key = false
if ($override_disk_size)
unless Vagrant.has_plugin?("vagrant-disksize")
system "vagrant plugin install vagrant-disksize"
end
config.disksize.size = $disk_size
end
(1..$num_instances).each do |i|
config.vm.define vm_name = "%s-%01d" % [$instance_name_prefix, i] do |node|
# First node would be master node
if i == 1
master = true
else
master = false
end
node.vm.hostname = vm_name
config.ssh.insert_key = false
vm_name = "%s-%02d" % [$instance_name_prefix, i]
if Vagrant.has_plugin?("vagrant-proxyconf")
node.proxy.http = ENV['HTTP_PROXY'] || ENV['http_proxy'] || ""
node.proxy.https = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || ""
node.proxy.no_proxy = $no_proxy
config.vm.define vm_name do |test_vm|
test_vm.vm.box = $box
test_vm.vm.hostname = vm_name
# Libvirt provider settings
test_vm.vm.provider :libvirt do |domain|
domain.uri = "qemu+unix:///system"
domain.memory = $vm_memory
domain.cpus = $vm_cpus
domain.driver = "kvm"
domain.host = "localhost"
domain.connect_via_ssh = false
domain.username = $user
domain.storage_pool_name = "default"
domain.nic_model_type = "e1000"
domain.management_network_name = "#{$instance_name_prefix}-mgmt-net"
domain.management_network_address = $mgmt_cidr
domain.nested = true
domain.cpu_mode = "host-passthrough"
domain.volume_cache = "unsafe"
domain.disk_bus = "virtio"
end
["vmware_fusion", "vmware_workstation"].each do |vmware|
node.vm.provider vmware do |v|
v.vmx['memsize'] = $vm_memory
v.vmx['numvcpus'] = $vm_cpus
end
end
node.vm.provider :virtualbox do |vb|
vb.memory = $vm_memory
vb.cpus = $vm_cpus
vb.gui = $vm_gui
vb.linked_clone = true
vb.customize ["modifyvm", :id, "--vram", "8"] # ubuntu defaults to 256 MB which is a waste of precious RAM
vb.customize ["modifyvm", :id, "--audio", "none"]
end
node.vm.provider :libvirt do |lv|
lv.nested = $libvirt_nested
lv.cpu_mode = "host-model"
lv.memory = $vm_memory
lv.cpus = $vm_cpus
lv.default_prefix = 'kubespray'
# Fix kernel panic on fedora 28
if $os == "fedora"
lv.cpu_mode = "host-passthrough"
end
end
if $kube_node_instances_with_disks
# Libvirt
driverletters = ('a'..'z').to_a
node.vm.provider :libvirt do |lv|
# always make /dev/sd{a/b/c} so that CI can ensure that
# virtualbox and libvirt will have the same devices to use for OSDs
(1..$kube_node_instances_with_disks_number).each do |d|
lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "scsi"
end
end
node.vm.provider :virtualbox do |vb|
# always make /dev/sd{a/b/c} so that CI can ensure that
# virtualbox and libvirt will have the same devices to use for OSDs
(1..$kube_node_instances_with_disks_number).each do |d|
vb.customize ['createhd', '--filename', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--size', $kube_node_instances_with_disks_size] # 10GB disk
vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', d, '--device', 0, '--type', 'hdd', '--medium', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--nonrotational', 'on', '--mtype', 'normal']
end
end
end
if $expose_docker_tcp
node.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true
end
$forwarded_ports.each do |guest, host|
node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
end
if ["rhel8"].include? $os
# Vagrant synced_folder rsync options cannot be used for RHEL boxes as Rsync package cannot
# be installed until the host is registered with a valid Red Hat support subscription
node.vm.synced_folder ".", "/vagrant", disabled: false
$shared_folders.each do |src, dst|
node.vm.synced_folder src, dst
end
else
node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv']
$shared_folders.each do |src, dst|
node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
end
end
ip = "#{$subnet}.#{i+100}"
ip6 = "#{$subnet_ipv6}::#{i+100}"
node.vm.network :private_network,
# Networks and interfaces
ip = "#{$private_subnet}.#{i+10}"
pub_ip = "#{$public_subnet}.#{i+10}"
# "public" network with nat forwarding
test_vm.vm.network :private_network,
:ip => pub_ip,
:model_type => "e1000",
:libvirt__network_name => "#{$instance_name_prefix}-public",
:libvirt__dhcp_enabled => false,
:libvirt__forward_mode => "nat"
# "private" isolated network
test_vm.vm.network :private_network,
:ip => ip,
:libvirt__guest_ipv6 => 'yes',
:libvirt__ipv6_address => ip6,
:libvirt__ipv6_prefix => "64",
:libvirt__forward_mode => "none",
:libvirt__dhcp_enabled => false
:model_type => "e1000",
:libvirt__network_name => "#{$instance_name_prefix}-private",
:libvirt__dhcp_enabled => false,
:libvirt__forward_mode => "none"
# libvirt__ipv6_address does not work as intended, the address is obtained with the desired prefix, but auto-generated(like fd3c:b398:698:756:5054:ff:fe48:c61e/64)
# add default route for detect ansible_default_ipv6
# TODO: fix libvirt__ipv6 or use $subnet in shell
config.vm.provision "shell", inline: "ip -6 r a fd3c:b398:698:756::/64 dev eth1;ip -6 r add default via fd3c:b398:0698:0756::1 dev eth1 || true"
# Disable swap for each vm
node.vm.provision "shell", inline: "swapoff -a"
# ubuntu2004 and ubuntu2204 have IPv6 explicitly disabled. This undoes that.
if ["ubuntu2004", "ubuntu2204"].include? $os
node.vm.provision "shell", inline: "rm -f /etc/modprobe.d/local.conf"
node.vm.provision "shell", inline: "sed -i '/net.ipv6.conf.all.disable_ipv6/d' /etc/sysctl.d/99-sysctl.conf /etc/sysctl.conf"
end
# Hack for fedora39/40 to get the IP address of the second interface
if ["fedora39", "fedora40", "fedora39-arm64", "fedora40-arm64"].include? $os
config.vm.provision "shell", inline: <<-SHELL
nmcli conn modify 'Wired connection 2' ipv4.addresses $(cat /etc/sysconfig/network-scripts/ifcfg-eth1 | grep IPADDR | cut -d "=" -f2)/24
nmcli conn modify 'Wired connection 2' ipv4.method manual
service NetworkManager restart
SHELL
end
# Rockylinux boxes needs UEFI
if ["rockylinux8", "rockylinux9"].include? $os
config.vm.provider "libvirt" do |domain|
domain.loader = "/usr/share/OVMF/x64/OVMF_CODE.fd"
end
end
# Disable firewalld on oraclelinux/redhat vms
if ["oraclelinux","oraclelinux8", "rhel8","rockylinux8"].include? $os
node.vm.provision "shell", inline: "systemctl stop firewalld; systemctl disable firewalld"
end
host_vars[vm_name] = {
"ip": ip,
"flannel_interface": "eth1",
"kube_network_plugin": $network_plugin,
"kube_network_plugin_multus": $multi_networking,
"download_run_once": $download_run_once,
"download_localhost": "False",
"download_cache_dir": ENV['HOME'] + "/kubespray_cache",
# Make kubespray cache even when download_run_once is false
"download_force_cache": $download_force_cache,
# Keeping the cache on the nodes can improve provisioning speed while debugging kubespray
"download_keep_remote_cache": "False",
"docker_rpm_keepcache": "1",
# These two settings will put kubectl and admin.config in $inventory/artifacts
"kubeconfig_localhost": "True",
"kubectl_localhost": "True",
"local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
"local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
"ansible_ssh_user": SUPPORTED_OS[$os][:user],
"ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"),
"unsafe_show_logs": "True"
}
# Only execute the Ansible provisioner once, when all the machines are up and ready.
# And limit the action to gathering facts, the full playbook is going to be ran by testcases_run.sh
if i == $num_instances
node.vm.provision "ansible" do |ansible|
ansible.playbook = $playbook
ansible.compatibility_mode = "2.0"
ansible.verbose = $ansible_verbosity
ansible.become = true
ansible.limit = "all,localhost"
ansible.host_key_checking = false
ansible.raw_arguments = ["--forks=#{$num_instances}",
"--flush-cache",
"-e ansible_become_pass=vagrant"] +
$inventories.map {|inv| ["-i", inv]}.flatten
ansible.host_vars = host_vars
ansible.extra_vars = $extra_vars
if $ansible_tags != ""
ansible.tags = [$ansible_tags]
end
ansible.groups = {
"etcd" => ["#{$instance_name_prefix}-[#{$first_etcd}:#{$etcd_instances + $first_etcd - 1}]"],
"kube_control_plane" => ["#{$instance_name_prefix}-[#{$first_control_plane}:#{$control_plane_instances + $first_control_plane - 1}]"],
"kube_node" => ["#{$instance_name_prefix}-[#{$first_node}:#{$kube_node_instances + $first_node - 1}]"],
"k8s_cluster:children" => ["kube_control_plane", "kube_node"],
}
end
# Provisioning
config.vm.provision "file", source: "ssh", destination: "~/ssh"
if master
config.vm.provision "deploy-k8s", type: "file", source: "deploy-k8s.kargo.sh", destination: "~/deploy-k8s.kargo.sh"
config.vm.provision "custom.yaml", type: "file", source: "custom.yaml", destination: "~/custom.yaml"
config.vm.provision "kubedns.yaml", type: "file", source: "kubedns.yaml", destination: "~/kubedns.yaml"
config.vm.provision "nodes", type: "file", source: "nodes", destination: "~/nodes"
config.vm.provision "bootstrap", type: "shell", path: "bootstrap-master.sh"
else
config.vm.provision "bootstrap", type: "shell", path: "bootstrap-node.sh"
end
end
-2
View File
@@ -1,2 +0,0 @@
---
theme: jekyll-theme-slate
-23
View File
@@ -1,23 +0,0 @@
[ssh_connection]
pipelining=True
ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
#control_path = ~/.ssh/ansible-%%r@%%h:%%p
[defaults]
# https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
force_valid_group_names = ignore
host_key_checking=False
gathering = smart
fact_caching = jsonfile
fact_caching_connection = /tmp
fact_caching_timeout = 86400
timeout = 300
stdout_callback = default
display_skipped_hosts = no
library = ./library
callbacks_enabled = profile_tasks,ara_default
roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
deprecation_warnings=False
inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg
[inventory]
ignore_patterns = artifacts, credentials
+31
View File
@@ -0,0 +1,31 @@
#!/bin/bash
echo master > /var/tmp/role
# Packages
sudo apt-get --yes update
sudo apt-get --yes upgrade
sudo apt-get --yes install git screen vim telnet tcpdump python-setuptools gcc python-dev python-pip libssl-dev libffi-dev software-properties-common
# Get ansible-2.1+, vanilla ubuntu-16.04 ansible (2.0.0.2) is broken due to https://github.com/ansible/ansible/issues/13876
sudo sh -c 'apt-add-repository -y ppa:ansible/ansible;apt-get update;apt-get install -y ansible'
# Kargo-cli
sudo git clone https://github.com/kubespray/kargo-cli.git /root/kargo-cli
sudo sh -c 'cd /root/kargo-cli && python setup.py install'
# k8s deploy script and configs
sudo sh -c 'cp -a ~vagrant/deploy-k8s.kargo.sh /root/ && chmod 755 /root/deploy-k8s.kargo.sh'
sudo cp -a ~vagrant/custom.yaml /root/custom.yaml
sudo cp -a ~vagrant/kubedns.yaml /root/kubedns.yaml
# SSH keys and config
sudo rm -rf /root/.ssh
sudo mv ~vagrant/ssh /root/.ssh
sudo echo -e 'Host 10.*\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null' >> /root/.ssh/config
sudo chown -R root: /root/.ssh
# Copy nodes list
sudo cp ~vagrant/nodes /root/nodes
# README
sudo echo 'cd /root/kargo ; ansible-playbook -vvv -i inv/inventory.cfg cluster.yml -u root -f 7' > /root/README
+17
View File
@@ -0,0 +1,17 @@
#!/bin/bash
echo node > /var/tmp/role
# Packages
sudo apt-get --yes update
sudo apt-get --yes upgrade
sudo apt-get --yes install screen vim telnet tcpdump python-pip traceroute iperf3 nmap ethtool
# Pip
sudo pip install kpm
# SSH
sudo rm -rf /root/.ssh
sudo mv ~vagrant/ssh /root/.ssh
sudo rm -f /root/.ssh/id_rsa*
sudo chown -R root: /root/.ssh
-3
View File
@@ -1,3 +0,0 @@
---
- name: Install Kubernetes
ansible.builtin.import_playbook: playbooks/cluster.yml
-3
View File
@@ -1,3 +0,0 @@
# Kubernetes Community Code of Conduct
Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md)
@@ -1,27 +0,0 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["ec2:*"],
"Resource": ["*"]
},
{
"Effect": "Allow",
"Action": ["elasticloadbalancing:*"],
"Resource": ["*"]
},
{
"Effect": "Allow",
"Action": ["route53:*"],
"Resource": ["*"]
},
{
"Effect": "Allow",
"Action": "s3:*",
"Resource": [
"arn:aws:s3:::kubernetes-*"
]
}
]
}
@@ -1,10 +0,0 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": { "Service": "ec2.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
@@ -1,45 +0,0 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "s3:*",
"Resource": [
"arn:aws:s3:::kubernetes-*"
]
},
{
"Effect": "Allow",
"Action": "ec2:Describe*",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "ec2:AttachVolume",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "ec2:DetachVolume",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": ["route53:*"],
"Resource": ["*"]
},
{
"Effect": "Allow",
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:GetRepositoryPolicy",
"ecr:DescribeRepositories",
"ecr:ListImages",
"ecr:BatchGetImage"
],
"Resource": "*"
}
]
}
@@ -1,10 +0,0 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": { "Service": "ec2.amazonaws.com"},
"Action": "sts:AssumeRole"
}
]
}
@@ -1,81 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import boto3
import os
import argparse
import json
class SearchEC2Tags(object):
def __init__(self):
self.parse_args()
if self.args.list:
self.search_tags()
if self.args.host:
data = {}
print(json.dumps(data, indent=2))
def parse_args(self):
##Check if VPC_VISIBILITY is set, if not default to private
if "VPC_VISIBILITY" in os.environ:
self.vpc_visibility = os.environ['VPC_VISIBILITY']
else:
self.vpc_visibility = "private"
##Support --list and --host flags. We largely ignore the host one.
parser = argparse.ArgumentParser()
parser.add_argument('--list', action='store_true', default=False, help='List instances')
parser.add_argument('--host', action='store_true', help='Get all the variables about a specific instance')
self.args = parser.parse_args()
def search_tags(self):
hosts = {}
hosts['_meta'] = { 'hostvars': {} }
##Search ec2 three times to find nodes of each group type. Relies on kubespray-role key/value.
for group in ["kube_control_plane", "kube_node", "etcd"]:
hosts[group] = []
tag_key = "kubespray-role"
tag_value = ["*"+group+"*"]
region = os.environ['AWS_REGION']
ec2 = boto3.resource('ec2', region)
filters = [{'Name': 'tag:'+tag_key, 'Values': tag_value}, {'Name': 'instance-state-name', 'Values': ['running']}]
cluster_name = os.getenv('CLUSTER_NAME')
if cluster_name:
filters.append({'Name': 'tag-key', 'Values': ['kubernetes.io/cluster/'+cluster_name]})
instances = ec2.instances.filter(Filters=filters)
for instance in instances:
##Suppose default vpc_visibility is private
dns_name = instance.private_dns_name
ansible_host = {
'ansible_ssh_host': instance.private_ip_address
}
##Override when vpc_visibility actually is public
if self.vpc_visibility == "public":
dns_name = instance.public_dns_name
ansible_host = {
'ansible_ssh_host': instance.public_ip_address
}
##Set when instance actually has node_labels
node_labels_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-labels', instance.tags))
if node_labels_tag:
ansible_host['node_labels'] = dict([ label.strip().split('=') for label in node_labels_tag[0]['Value'].split(',') ])
##Set when instance actually has node_taints
node_taints_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-taints', instance.tags))
if node_taints_tag:
ansible_host['node_taints'] = list([ taint.strip() for taint in node_taints_tag[0]['Value'].split(',') ])
hosts[group].append(dns_name)
hosts['_meta']['hostvars'][dns_name] = ansible_host
hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']}
print(json.dumps(hosts, sort_keys=True, indent=2))
SearchEC2Tags()
-1
View File
@@ -1 +0,0 @@
boto3 # Apache-2.0
-2
View File
@@ -1,2 +0,0 @@
.generated
/inventory
-67
View File
@@ -1,67 +0,0 @@
# Kubernetes on Azure with Azure Resource Group Templates
Provision the base infrastructure for a Kubernetes cluster by using [Azure Resource Group Templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-authoring-templates)
## Status
This will provision the base infrastructure (vnet, vms, nics, ips, ...) needed for Kubernetes in Azure into the specified
Resource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kubespray of course).
## Requirements
- [Install azure-cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest)
- [Login with azure-cli](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest)
- Dedicated Resource Group created in the Azure Portal or through azure-cli
## Configuration through group_vars/all
You have to modify at least two variables in group_vars/all. The one is the **cluster_name** variable, it must be globally
unique due to some restrictions in Azure. The other one is the **ssh_public_keys** variable, it must be your ssh public
key to access your azure virtual machines. Most other variables should be self explanatory if you have some basic Kubernetes
experience.
## Bastion host
You can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated
templates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option
also removes all public IPs from all other VMs.
## Generating and applying
To generate and apply the templates, call:
```shell
./apply-rg.sh <resource_group_name>
```
If you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will
take care about creating/modifying whatever is needed.
## Clearing a resource group
If you need to delete all resources from a resource group, simply call:
```shell
./clear-rg.sh <resource_group_name>
```
**WARNING** this really deletes everything from your resource group, including everything that was later created by you!
## Installing Ansible and the dependencies
Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
## Generating an inventory for kubespray
After you have applied the templates, you can generate an inventory with this call:
```shell
./generate-inventory.sh <resource_group_name>
```
It will create the file ./inventory which can then be used with kubespray, e.g.:
```shell
cd kubespray-root-dir
ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/sample/group_vars/all/all.yml" cluster.yml
```
-19
View File
@@ -1,19 +0,0 @@
#!/usr/bin/env bash
set -e
AZURE_RESOURCE_GROUP="$1"
if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
echo "AZURE_RESOURCE_GROUP is missing"
exit 1
fi
ansible-playbook generate-templates.yml
az deployment group create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP
az deployment group create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
az deployment group create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
az deployment group create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
az deployment group create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
az deployment group create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP
-14
View File
@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -e
AZURE_RESOURCE_GROUP="$1"
if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
echo "AZURE_RESOURCE_GROUP is missing"
exit 1
fi
ansible-playbook generate-templates.yml
az group deployment create -g "$AZURE_RESOURCE_GROUP" --template-file ./.generated/clear-rg.json --mode Complete
-18
View File
@@ -1,18 +0,0 @@
#!/usr/bin/env bash
set -e
AZURE_RESOURCE_GROUP="$1"
if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
echo "AZURE_RESOURCE_GROUP is missing"
exit 1
fi
# check if azure cli 2.0 exists else use azure cli 1.0
if az &>/dev/null; then
ansible-playbook generate-inventory_2.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
elif azure &>/dev/null; then
ansible-playbook generate-inventory.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
else
echo "Azure cli not found"
fi
-6
View File
@@ -1,6 +0,0 @@
---
- name: Generate Azure inventory
hosts: localhost
gather_facts: false
roles:
- generate-inventory
-6
View File
@@ -1,6 +0,0 @@
---
- name: Generate Azure inventory
hosts: localhost
gather_facts: false
roles:
- generate-inventory_2
-6
View File
@@ -1,6 +0,0 @@
---
- name: Generate Azure templates
hosts: localhost
gather_facts: false
roles:
- generate-templates
-51
View File
@@ -1,51 +0,0 @@
# Due to some Azure limitations (ex:- Storage Account's name must be unique),
# this name must be globally unique - it will be used as a prefix for azure components
cluster_name: example
# Set this to true if you do not want to have public IPs for your masters and minions. This will provision a bastion
# node that can be used to access the masters and minions
use_bastion: false
# Set this to a preferred name that will be used as the first part of the dns name for your bastotion host. For example: k8s-bastion.<azureregion>.cloudapp.azure.com.
# This is convenient when exceptions have to be configured on a firewall to allow ssh to the given bastion host.
# bastion_domain_prefix: k8s-bastion
number_of_k8s_masters: 3
number_of_k8s_nodes: 3
masters_vm_size: Standard_A2
masters_os_disk_size: 1000
minions_vm_size: Standard_A2
minions_os_disk_size: 1000
admin_username: devops
admin_password: changeme
# MAKE SURE TO CHANGE THIS TO YOUR PUBLIC KEY to access your azure machines
ssh_public_keys:
- "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy"
# Disable using ssh using password. Change it to false to allow to connect to ssh by password
disablePasswordAuthentication: true
# Azure CIDRs
azure_vnet_cidr: 10.0.0.0/8
azure_admin_cidr: 10.241.2.0/24
azure_masters_cidr: 10.0.4.0/24
azure_minions_cidr: 10.240.0.0/16
# Azure loadbalancer port to use to access your cluster
kube_apiserver_port: 6443
# Azure Netwoking and storage naming to use with inventory/all.yml
#azure_virtual_network_name: KubeVNET
#azure_subnet_admin_name: ad-subnet
#azure_subnet_masters_name: master-subnet
#azure_subnet_minions_name: minion-subnet
#azure_route_table_name: routetable
#azure_security_group_name: secgroup
# Storage types available are: "Standard_LRS","Premium_LRS"
#azure_storage_account_type: Standard_LRS
@@ -1,15 +0,0 @@
---
- name: Query Azure VMs
command: azure vm list-ip-address --json {{ azure_resource_group }}
register: vm_list_cmd
- name: Set vm_list
set_fact:
vm_list: "{{ vm_list_cmd.stdout }}"
- name: Generate inventory
template:
src: inventory.j2
dest: "{{ playbook_dir }}/inventory"
mode: "0644"
@@ -1,33 +0,0 @@
{% for vm in vm_list %}
{% if not use_bastion or vm.name == 'bastion' %}
{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].publicIPAddress.expanded.ipAddress }} ip={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
{% else %}
{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
{% endif %}
{% endfor %}
[kube_control_plane]
{% for vm in vm_list %}
{% if 'kube_control_plane' in vm.tags.roles %}
{{ vm.name }}
{% endif %}
{% endfor %}
[etcd]
{% for vm in vm_list %}
{% if 'etcd' in vm.tags.roles %}
{{ vm.name }}
{% endif %}
{% endfor %}
[kube_node]
{% for vm in vm_list %}
{% if 'kube_node' in vm.tags.roles %}
{{ vm.name }}
{% endif %}
{% endfor %}
[k8s_cluster:children]
kube_node
kube_control_plane
@@ -1,31 +0,0 @@
---
- name: Query Azure VMs IPs
command: az vm list-ip-addresses -o json --resource-group {{ azure_resource_group }}
register: vm_ip_list_cmd
- name: Query Azure VMs Roles
command: az vm list -o json --resource-group {{ azure_resource_group }}
register: vm_list_cmd
- name: Query Azure Load Balancer Public IP
command: az network public-ip show -o json -g {{ azure_resource_group }} -n kubernetes-api-pubip
register: lb_pubip_cmd
- name: Set VM IP, roles lists and load balancer public IP
set_fact:
vm_ip_list: "{{ vm_ip_list_cmd.stdout }}"
vm_roles_list: "{{ vm_list_cmd.stdout }}"
lb_pubip: "{{ lb_pubip_cmd.stdout }}"
- name: Generate inventory
template:
src: inventory.j2
dest: "{{ playbook_dir }}/inventory"
mode: "0644"
- name: Generate Load Balancer variables
template:
src: loadbalancer_vars.j2
dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
mode: "0644"
@@ -1,33 +0,0 @@
{% for vm in vm_ip_list %}
{% if not use_bastion or vm.virtualMachine.name == 'bastion' %}
{{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.publicIpAddresses[0].ipAddress }} ip={{ vm.virtualMachine.network.privateIpAddresses[0] }}
{% else %}
{{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.privateIpAddresses[0] }}
{% endif %}
{% endfor %}
[kube_control_plane]
{% for vm in vm_roles_list %}
{% if 'kube_control_plane' in vm.tags.roles %}
{{ vm.name }}
{% endif %}
{% endfor %}
[etcd]
{% for vm in vm_roles_list %}
{% if 'etcd' in vm.tags.roles %}
{{ vm.name }}
{% endif %}
{% endfor %}
[kube_node]
{% for vm in vm_roles_list %}
{% if 'kube_node' in vm.tags.roles %}
{{ vm.name }}
{% endif %}
{% endfor %}
[k8s_cluster:children]
kube_node
kube_control_plane
@@ -1,8 +0,0 @@
## External LB example config
apiserver_loadbalancer_domain_name: {{ lb_pubip.dnsSettings.fqdn }}
loadbalancer_apiserver:
address: {{ lb_pubip.ipAddress }}
port: 6443
## Internal loadbalancers for apiservers
loadbalancer_apiserver_localhost: false
@@ -1,37 +0,0 @@
---
apiVersion: "2015-06-15"
virtualNetworkName: "{{ azure_virtual_network_name | default('KubeVNET') }}"
subnetAdminName: "{{ azure_subnet_admin_name | default('ad-subnet') }}"
subnetMastersName: "{{ azure_subnet_masters_name | default('master-subnet') }}"
subnetMinionsName: "{{ azure_subnet_minions_name | default('minion-subnet') }}"
routeTableName: "{{ azure_route_table_name | default('routetable') }}"
securityGroupName: "{{ azure_security_group_name | default('secgroup') }}"
nameSuffix: "{{ cluster_name }}"
availabilitySetMasters: "master-avs"
availabilitySetMinions: "minion-avs"
faultDomainCount: 3
updateDomainCount: 10
bastionVmSize: Standard_A0
bastionVMName: bastion
bastionIPAddressName: bastion-pubip
disablePasswordAuthentication: true
sshKeyPath: "/home/{{ admin_username }}/.ssh/authorized_keys"
imageReference:
publisher: "OpenLogic"
offer: "CentOS"
sku: "7.5"
version: "latest"
imageReferenceJson: "{{ imageReference | to_json }}"
storageAccountName: "sa{{ nameSuffix | replace('-', '') }}"
storageAccountType: "{{ azure_storage_account_type | default('Standard_LRS') }}"
@@ -1,25 +0,0 @@
---
- name: Set base_dir
set_fact:
base_dir: "{{ playbook_dir }}/.generated/"
- name: Create base_dir
file:
path: "{{ base_dir }}"
state: directory
recurse: true
mode: "0755"
- name: Store json files in base_dir
template:
src: "{{ item }}"
dest: "{{ base_dir }}/{{ item }}"
mode: "0644"
with_items:
- network.json
- storage.json
- availability-sets.json
- bastion.json
- masters.json
- minions.json
- clear-rg.json
@@ -1,30 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
},
"variables": {
},
"resources": [
{
"type": "Microsoft.Compute/availabilitySets",
"name": "{{availabilitySetMasters}}",
"apiVersion": "{{apiVersion}}",
"location": "[resourceGroup().location]",
"properties": {
"PlatformFaultDomainCount": "{{faultDomainCount}}",
"PlatformUpdateDomainCount": "{{updateDomainCount}}"
}
},
{
"type": "Microsoft.Compute/availabilitySets",
"name": "{{availabilitySetMinions}}",
"apiVersion": "{{apiVersion}}",
"location": "[resourceGroup().location]",
"properties": {
"PlatformFaultDomainCount": "{{faultDomainCount}}",
"PlatformUpdateDomainCount": "{{updateDomainCount}}"
}
}
]
}
@@ -1,106 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
},
"variables": {
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
"subnetAdminRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetAdminName}}')]"
},
"resources": [
{% if use_bastion %}
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/publicIPAddresses",
"name": "{{bastionIPAddressName}}",
"location": "[resourceGroup().location]",
"properties": {
"publicIPAllocationMethod": "Static",
"dnsSettings": {
{% if bastion_domain_prefix %}
"domainNameLabel": "{{ bastion_domain_prefix }}"
{% endif %}
}
}
},
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/networkInterfaces",
"name": "{{bastionVMName}}-nic",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/publicIPAddresses/', '{{bastionIPAddressName}}')]"
],
"properties": {
"ipConfigurations": [
{
"name": "BastionIpConfig",
"properties": {
"privateIPAllocationMethod": "Dynamic",
"publicIPAddress": {
"id": "[resourceId('Microsoft.Network/publicIPAddresses', '{{bastionIPAddressName}}')]"
},
"subnet": {
"id": "[variables('subnetAdminRef')]"
}
}
}
]
}
},
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Compute/virtualMachines",
"name": "{{bastionVMName}}",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/networkInterfaces/', '{{bastionVMName}}-nic')]"
],
"tags": {
"roles": "bastion"
},
"properties": {
"hardwareProfile": {
"vmSize": "{{bastionVmSize}}"
},
"osProfile": {
"computerName": "{{bastionVMName}}",
"adminUsername": "{{admin_username}}",
"adminPassword": "{{admin_password}}",
"linuxConfiguration": {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{% for key in ssh_public_keys %}
{
"path": "{{sshKeyPath}}",
"keyData": "{{key}}"
}{% if loop.index < ssh_public_keys | length %},{% endif %}
{% endfor %}
]
}
}
},
"storageProfile": {
"imageReference": {{imageReferenceJson}},
"osDisk": {
"name": "osdisk",
"vhd": {
"uri": "[concat('http://', '{{storageAccountName}}', '.blob.core.windows.net/vhds/', '{{bastionVMName}}', '-osdisk.vhd')]"
},
"caching": "ReadWrite",
"createOption": "FromImage"
}
},
"networkProfile": {
"networkInterfaces": [
{
"id": "[resourceId('Microsoft.Network/networkInterfaces', '{{bastionVMName}}-nic')]"
}
]
}
}
}
{% endif %}
]
}
@@ -1,8 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {},
"variables": {},
"resources": [],
"outputs": {}
}
@@ -1,198 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
},
"variables": {
"lbDomainName": "{{nameSuffix}}-api",
"lbPublicIPAddressName": "kubernetes-api-pubip",
"lbPublicIPAddressType": "Static",
"lbPublicIPAddressID": "[resourceId('Microsoft.Network/publicIPAddresses',variables('lbPublicIPAddressName'))]",
"lbName": "kubernetes-api",
"lbID": "[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]",
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
"kubeMastersSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMastersName}}')]"
},
"resources": [
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/publicIPAddresses",
"name": "[variables('lbPublicIPAddressName')]",
"location": "[resourceGroup().location]",
"properties": {
"publicIPAllocationMethod": "[variables('lbPublicIPAddressType')]",
"dnsSettings": {
"domainNameLabel": "[variables('lbDomainName')]"
}
}
},
{
"apiVersion": "{{apiVersion}}",
"name": "[variables('lbName')]",
"type": "Microsoft.Network/loadBalancers",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/publicIPAddresses/', variables('lbPublicIPAddressName'))]"
],
"properties": {
"frontendIPConfigurations": [
{
"name": "kube-api-frontend",
"properties": {
"publicIPAddress": {
"id": "[variables('lbPublicIPAddressID')]"
}
}
}
],
"backendAddressPools": [
{
"name": "kube-api-backend"
}
],
"loadBalancingRules": [
{
"name": "kube-api",
"properties": {
"frontendIPConfiguration": {
"id": "[concat(variables('lbID'), '/frontendIPConfigurations/kube-api-frontend')]"
},
"backendAddressPool": {
"id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
},
"protocol": "tcp",
"frontendPort": "{{kube_apiserver_port}}",
"backendPort": "{{kube_apiserver_port}}",
"enableFloatingIP": false,
"idleTimeoutInMinutes": 5,
"probe": {
"id": "[concat(variables('lbID'), '/probes/kube-api')]"
}
}
}
],
"probes": [
{
"name": "kube-api",
"properties": {
"protocol": "tcp",
"port": "{{kube_apiserver_port}}",
"intervalInSeconds": 5,
"numberOfProbes": 2
}
}
]
}
},
{% for i in range(number_of_k8s_masters) %}
{% if not use_bastion %}
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/publicIPAddresses",
"name": "master-{{i}}-pubip",
"location": "[resourceGroup().location]",
"properties": {
"publicIPAllocationMethod": "Static"
}
},
{% endif %}
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/networkInterfaces",
"name": "master-{{i}}-nic",
"location": "[resourceGroup().location]",
"dependsOn": [
{% if not use_bastion %}
"[concat('Microsoft.Network/publicIPAddresses/', 'master-{{i}}-pubip')]",
{% endif %}
"[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]"
],
"properties": {
"ipConfigurations": [
{
"name": "MastersIpConfig",
"properties": {
"privateIPAllocationMethod": "Dynamic",
{% if not use_bastion %}
"publicIPAddress": {
"id": "[resourceId('Microsoft.Network/publicIPAddresses', 'master-{{i}}-pubip')]"
},
{% endif %}
"subnet": {
"id": "[variables('kubeMastersSubnetRef')]"
},
"loadBalancerBackendAddressPools": [
{
"id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
}
]
}
}
],
"networkSecurityGroup": {
"id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
},
"enableIPForwarding": true
}
},
{
"type": "Microsoft.Compute/virtualMachines",
"name": "master-{{i}}",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]"
],
"tags": {
"roles": "kube_control_plane,etcd"
},
"apiVersion": "{{apiVersion}}",
"properties": {
"availabilitySet": {
"id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMasters}}')]"
},
"hardwareProfile": {
"vmSize": "{{masters_vm_size}}"
},
"osProfile": {
"computerName": "master-{{i}}",
"adminUsername": "{{admin_username}}",
"adminPassword": "{{admin_password}}",
"linuxConfiguration": {
"disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
"ssh": {
"publicKeys": [
{% for key in ssh_public_keys %}
{
"path": "{{sshKeyPath}}",
"keyData": "{{key}}"
}{% if loop.index < ssh_public_keys | length %},{% endif %}
{% endfor %}
]
}
}
},
"storageProfile": {
"imageReference": {{imageReferenceJson}},
"osDisk": {
"name": "ma{{nameSuffix}}{{i}}",
"vhd": {
"uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/master-{{i}}.vhd')]"
},
"caching": "ReadWrite",
"createOption": "FromImage",
"diskSizeGB": "{{masters_os_disk_size}}"
}
},
"networkProfile": {
"networkInterfaces": [
{
"id": "[resourceId('Microsoft.Network/networkInterfaces', 'master-{{i}}-nic')]"
}
]
}
}
} {% if not loop.last %},{% endif %}
{% endfor %}
]
}
@@ -1,115 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
},
"variables": {
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
"kubeMinionsSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMinionsName}}')]"
},
"resources": [
{% for i in range(number_of_k8s_nodes) %}
{% if not use_bastion %}
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/publicIPAddresses",
"name": "minion-{{i}}-pubip",
"location": "[resourceGroup().location]",
"properties": {
"publicIPAllocationMethod": "Static"
}
},
{% endif %}
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/networkInterfaces",
"name": "minion-{{i}}-nic",
"location": "[resourceGroup().location]",
"dependsOn": [
{% if not use_bastion %}
"[concat('Microsoft.Network/publicIPAddresses/', 'minion-{{i}}-pubip')]"
{% endif %}
],
"properties": {
"ipConfigurations": [
{
"name": "MinionsIpConfig",
"properties": {
"privateIPAllocationMethod": "Dynamic",
{% if not use_bastion %}
"publicIPAddress": {
"id": "[resourceId('Microsoft.Network/publicIPAddresses', 'minion-{{i}}-pubip')]"
},
{% endif %}
"subnet": {
"id": "[variables('kubeMinionsSubnetRef')]"
}
}
}
],
"networkSecurityGroup": {
"id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
},
"enableIPForwarding": true
}
},
{
"type": "Microsoft.Compute/virtualMachines",
"name": "minion-{{i}}",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]"
],
"tags": {
"roles": "kube_node"
},
"apiVersion": "{{apiVersion}}",
"properties": {
"availabilitySet": {
"id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMinions}}')]"
},
"hardwareProfile": {
"vmSize": "{{minions_vm_size}}"
},
"osProfile": {
"computerName": "minion-{{i}}",
"adminUsername": "{{admin_username}}",
"adminPassword": "{{admin_password}}",
"linuxConfiguration": {
"disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
"ssh": {
"publicKeys": [
{% for key in ssh_public_keys %}
{
"path": "{{sshKeyPath}}",
"keyData": "{{key}}"
}{% if loop.index < ssh_public_keys | length %},{% endif %}
{% endfor %}
]
}
}
},
"storageProfile": {
"imageReference": {{imageReferenceJson}},
"osDisk": {
"name": "mi{{nameSuffix}}{{i}}",
"vhd": {
"uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/minion-{{i}}.vhd')]"
},
"caching": "ReadWrite",
"createOption": "FromImage",
"diskSizeGB": "{{minions_os_disk_size}}"
}
},
"networkProfile": {
"networkInterfaces": [
{
"id": "[resourceId('Microsoft.Network/networkInterfaces', 'minion-{{i}}-nic')]"
}
]
}
}
} {% if not loop.last %},{% endif %}
{% endfor %}
]
}
@@ -1,109 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
},
"variables": {
},
"resources": [
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/routeTables",
"name": "{{routeTableName}}",
"location": "[resourceGroup().location]",
"properties": {
"routes": [
]
}
},
{
"type": "Microsoft.Network/virtualNetworks",
"name": "{{virtualNetworkName}}",
"location": "[resourceGroup().location]",
"apiVersion": "{{apiVersion}}",
"dependsOn": [
"[concat('Microsoft.Network/routeTables/', '{{routeTableName}}')]"
],
"properties": {
"addressSpace": {
"addressPrefixes": [
"{{azure_vnet_cidr}}"
]
},
"subnets": [
{
"name": "{{subnetMastersName}}",
"properties": {
"addressPrefix": "{{azure_masters_cidr}}",
"routeTable": {
"id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
}
}
},
{
"name": "{{subnetMinionsName}}",
"properties": {
"addressPrefix": "{{azure_minions_cidr}}",
"routeTable": {
"id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
}
}
}
{% if use_bastion %}
,{
"name": "{{subnetAdminName}}",
"properties": {
"addressPrefix": "{{azure_admin_cidr}}",
"routeTable": {
"id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
}
}
}
{% endif %}
]
}
},
{
"apiVersion": "{{apiVersion}}",
"type": "Microsoft.Network/networkSecurityGroups",
"name": "{{securityGroupName}}",
"location": "[resourceGroup().location]",
"properties": {
"securityRules": [
{% if not use_bastion %}
{
"name": "ssh",
"properties": {
"description": "Allow SSH",
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "22",
"sourceAddressPrefix": "Internet",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 100,
"direction": "Inbound"
}
},
{% endif %}
{
"name": "kube-api",
"properties": {
"description": "Allow secure kube-api",
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "{{kube_apiserver_port}}",
"sourceAddressPrefix": "Internet",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 101,
"direction": "Inbound"
}
}
]
},
"resources": [],
"dependsOn": []
}
]
}
@@ -1,19 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
},
"variables": {
},
"resources": [
{
"type": "Microsoft.Storage/storageAccounts",
"name": "{{storageAccountName}}",
"location": "[resourceGroup().location]",
"apiVersion": "{{apiVersion}}",
"properties": {
"accountType": "{{storageAccountType}}"
}
}
]
}
-11
View File
@@ -1,11 +0,0 @@
# Kubespray on KVM Virtual Machines hypervisor preparation
A simple playbook to ensure your system has the right settings to enable Kubespray
deployment on VMs.
This playbook does not create Virtual Machines, nor does it run Kubespray itself.
## User creation
If you want to create a user for running Kubespray deployment, you should specify
both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.
-2
View File
@@ -1,2 +0,0 @@
#k8s_deployment_user: kubespray
#k8s_deployment_user_pkey_path: /tmp/ssh_rsa
-9
View File
@@ -1,9 +0,0 @@
---
- name: Prepare Hypervisor to later install kubespray VMs
hosts: localhost
gather_facts: false
become: true
vars:
bootstrap_os: none
roles:
- { role: kvm-setup }
@@ -1,30 +0,0 @@
---
- name: Install required packages
package:
name: "{{ item }}"
state: present
with_items:
- bind-utils
- ntp
when: ansible_os_family == "RedHat"
- name: Install required packages
apt:
upgrade: true
update_cache: true
cache_valid_time: 3600
name: "{{ item }}"
state: present
install_recommends: false
with_items:
- dnsutils
- ntp
when: ansible_os_family == "Debian"
- name: Create deployment user if required
include_tasks: user.yml
when: k8s_deployment_user is defined
- name: Set proper sysctl values
import_tasks: sysctl.yml
@@ -1,46 +0,0 @@
---
- name: Load br_netfilter module
community.general.modprobe:
name: br_netfilter
state: present
register: br_netfilter
- name: Add br_netfilter into /etc/modules
lineinfile:
dest: /etc/modules
state: present
line: 'br_netfilter'
when: br_netfilter is defined and ansible_os_family == 'Debian'
- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
copy:
dest: /etc/modules-load.d/kubespray.conf
content: |-
### This file is managed by Ansible
br-netfilter
owner: root
group: root
mode: "0644"
when: br_netfilter is defined
- name: Enable net.ipv4.ip_forward in sysctl
ansible.posix.sysctl:
name: net.ipv4.ip_forward
value: 1
sysctl_file: "{{ sysctl_file_path }}"
state: present
reload: true
- name: Set bridge-nf-call-{arptables,iptables} to 0
ansible.posix.sysctl:
name: "{{ item }}"
state: present
value: 0
sysctl_file: "{{ sysctl_file_path }}"
reload: true
with_items:
- net.bridge.bridge-nf-call-arptables
- net.bridge.bridge-nf-call-ip6tables
- net.bridge.bridge-nf-call-iptables
when: br_netfilter is defined
@@ -1,47 +0,0 @@
---
- name: Create user {{ k8s_deployment_user }}
user:
name: "{{ k8s_deployment_user }}"
groups: adm
shell: /bin/bash
- name: Ensure that .ssh exists
file:
path: "/home/{{ k8s_deployment_user }}/.ssh"
state: directory
owner: "{{ k8s_deployment_user }}"
group: "{{ k8s_deployment_user }}"
mode: "0700"
- name: Configure sudo for deployment user
copy:
content: |
%{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
dest: "/etc/sudoers.d/55-k8s-deployment"
owner: root
group: root
mode: "0644"
- name: Write private SSH key
copy:
src: "{{ k8s_deployment_user_pkey_path }}"
dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
mode: "0400"
owner: "{{ k8s_deployment_user }}"
group: "{{ k8s_deployment_user }}"
when: k8s_deployment_user_pkey_path is defined
- name: Write public SSH key
shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
> /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
args:
creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
when: k8s_deployment_user_pkey_path is defined
- name: Fix ssh-pub-key permissions
file:
path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
mode: "0600"
owner: "{{ k8s_deployment_user }}"
group: "{{ k8s_deployment_user }}"
when: k8s_deployment_user_pkey_path is defined
-15
View File
@@ -1,15 +0,0 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard
labels:
k8s-app: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kube-system
-51
View File
@@ -1,51 +0,0 @@
---
- name: Check ansible version
import_playbook: kubernetes_sigs.kubespray.ansible_version
- name: Install mitogen
hosts: localhost
strategy: linear
vars:
mitogen_version: 0.3.2
mitogen_url: https://github.com/mitogen-hq/mitogen/archive/refs/tags/v{{ mitogen_version }}.tar.gz
ansible_connection: local
tasks:
- name: Create mitogen plugin dir
file:
path: "{{ item }}"
state: directory
mode: "0755"
become: false
loop:
- "{{ playbook_dir }}/plugins/mitogen"
- "{{ playbook_dir }}/dist"
- name: Download mitogen release
get_url:
url: "{{ mitogen_url }}"
dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
validate_certs: true
mode: "0644"
- name: Extract archive
unarchive:
src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
dest: "{{ playbook_dir }}/dist/"
- name: Copy plugin
ansible.posix.synchronize:
src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
dest: "{{ playbook_dir }}/plugins/mitogen"
- name: Add strategy to ansible.cfg
community.general.ini_file:
path: ansible.cfg
mode: "0644"
section: "{{ item.section | d('defaults') }}"
option: "{{ item.option }}"
value: "{{ item.value }}"
with_items:
- option: strategy
value: mitogen_linear
- option: strategy_plugins
value: plugins/mitogen/ansible_mitogen/plugins/strategy
-89
View File
@@ -1,89 +0,0 @@
# Offline deployment
## manage-offline-container-images.sh
Container image collecting script for offline deployment
This script has two features:
(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE
environment variable to get images from a file (e.g. temp/images.list after running the
./generate_list.sh script).
(2) Deploy local container registry and register the container images to the registry.
Step(1) should be done online site as a preparation, then we bring the gotten images
to the target offline environment. if images are from a private registry,
you need to set `PRIVATE_REGISTRY` environment variable.
Then we will run step(2) for registering the images to local registry, or to an existing
registry set by the `DESTINATION_REGISTRY` environment variable. By default, the local registry
will run on port 5000. This can be changed with the `REGISTRY_PORT` environment variable
Step(1) can be operated with:
```shell
manage-offline-container-images.sh create
```
Step(2) can be operated with:
```shell
manage-offline-container-images.sh register
```
## generate_list.sh
This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/defaults/main/download.yml` file.
Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.
```shell
./generate_list.sh
tree temp
temp
├── files.list
├── files.list.template
├── images.list
└── images.list.template
0 directories, 5 files
```
In some cases you may want to update some component version, you can declare version variables in ansible inventory file or group_vars,
then run `./generate_list.sh -i [inventory_file]` to update file.list and images.list.
## manage-offline-files.sh
This script will download all files according to `temp/files.list` and run nginx container to provide offline file download.
Step(1) generate `files.list`
```shell
./generate_list.sh
```
Step(2) download files and run nginx container
```shell
./manage-offline-files.sh
```
when nginx container is running, it can be accessed through <http://127.0.0.1:8080/>.
## upload2artifactory.py
After the steps above, this script can recursively upload each file under a directory to a generic repository in Artifactory.
Environment Variables:
- USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'.
- TOKEN -- Generate this with 'Set Me Up' in your user.
- BASE_URL -- The URL including the repository name.
Step(3) (optional) upload files to Artifactory
```shell
cd kubespray/contrib/offline/offline-files
export USERNAME=admin
export TOKEN=...
export BASE_URL=https://artifactory.example.com/artifactory/a-generic-repo/
./upload2artifactory.py
```
-1
View File
@@ -1 +0,0 @@
{ "insecure-registries":["HOSTNAME:5000"] }
-33
View File
@@ -1,33 +0,0 @@
#!/bin/bash
set -eo pipefail
CURRENT_DIR=$(cd $(dirname $0); pwd)
TEMP_DIR="${CURRENT_DIR}/temp"
REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"
: ${DOWNLOAD_YML:="roles/kubespray-defaults/defaults/main/download.yml"}
mkdir -p ${TEMP_DIR}
# generate all download files url template
grep 'download_url:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
| sed 's/^.*_url: //g;s/\"//g' > ${TEMP_DIR}/files.list.template
# generate all images list template
sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
| sed -n "s/repo: //p;s/tag: //p" | tr -d ' ' \
| sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template
# add kube-* images to images list template
# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/defaults/main/download.yml
# doesn't contain those images. That is reason why here needs to put those images into the
# list separately.
KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"
for i in $KUBE_IMAGES; do
echo "{{ kube_image_repo }}/$i:v{{ kube_version }}" >> ${TEMP_DIR}/images.list.template
done
# run ansible to expand templates
/bin/cp ${CURRENT_DIR}/generate_list.yml ${REPO_ROOT_DIR}
(cd ${REPO_ROOT_DIR} && ansible-playbook $* generate_list.yml && /bin/rm generate_list.yml) || exit 1
-22
View File
@@ -1,22 +0,0 @@
---
- name: Collect container images for offline deployment
hosts: localhost
become: false
roles:
# Just load default variables from roles.
- role: kubespray-defaults
when: false
- role: download
when: false
tasks:
# Generate files.list and images.list files from templates.
- name: Collect container images for offline deployment
template:
src: ./contrib/offline/temp/{{ item }}.list.template
dest: ./contrib/offline/temp/{{ item }}.list
mode: "0644"
with_items:
- files
- images
@@ -1,223 +0,0 @@
#!/usr/bin/env bash
OPTION=$1
CURRENT_DIR=$(cd $(dirname $0); pwd)
TEMP_DIR="${CURRENT_DIR}/temp"
IMAGE_TAR_FILE="${CURRENT_DIR}/container-images.tar.gz"
IMAGE_DIR="${CURRENT_DIR}/container-images"
IMAGE_LIST="${IMAGE_DIR}/container-images.txt"
RETRY_COUNT=5
function create_container_image_tar() {
set -e
if [ -z "${IMAGES_FROM_FILE}" ]; then
echo "Getting images from current \"$(kubectl config current-context)\""
IMAGES=$(mktemp --suffix=-images)
trap 'rm -f "${IMAGES}"' EXIT
kubectl describe cronjobs,jobs,pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq > "${IMAGES}"
# NOTE: etcd and pause cannot be seen as pods.
# The pause image is used for --pod-infra-container-image option of kubelet.
kubectl cluster-info dump | grep -E "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g >> "${IMAGES}"
else
echo "Getting images from file \"${IMAGES_FROM_FILE}\""
if [ ! -f "${IMAGES_FROM_FILE}" ]; then
echo "${IMAGES_FROM_FILE} is not a file"
exit 1
fi
IMAGES=$(realpath $IMAGES_FROM_FILE)
fi
rm -f ${IMAGE_TAR_FILE}
rm -rf ${IMAGE_DIR}
mkdir ${IMAGE_DIR}
cd ${IMAGE_DIR}
sudo ${runtime} pull registry:latest
sudo ${runtime} save -o registry-latest.tar registry:latest
while read -r image
do
FILE_NAME="$(echo ${image} | sed s@"/"@"-"@g | sed s/":"/"-"/g | sed -E 's/\@.*//g')".tar
set +e
for step in $(seq 1 ${RETRY_COUNT})
do
sudo ${runtime} pull ${image}
if [ $? -eq 0 ]; then
break
fi
echo "Failed to pull ${image} at step ${step}"
if [ ${step} -eq ${RETRY_COUNT} ]; then
exit 1
fi
done
set -e
sudo ${runtime} save -o ${FILE_NAME} ${image}
# NOTE: Here removes the following repo parts from each image
# so that these parts will be replaced with Kubespray.
# - kube_image_repo: "registry.k8s.io"
# - gcr_image_repo: "gcr.io"
# - ghcr_image_repo: "ghcr.io"
# - docker_image_repo: "docker.io"
# - quay_image_repo: "quay.io"
FIRST_PART=$(echo ${image} | awk -F"/" '{print $1}')
if [ "${FIRST_PART}" = "registry.k8s.io" ] ||
[ "${FIRST_PART}" = "gcr.io" ] ||
[ "${FIRST_PART}" = "ghcr.io" ] ||
[ "${FIRST_PART}" = "docker.io" ] ||
[ "${FIRST_PART}" = "quay.io" ] ||
[ "${FIRST_PART}" = "${PRIVATE_REGISTRY}" ]; then
image=$(echo ${image} | sed s@"${FIRST_PART}/"@@ | sed -E 's/\@.*/\n/g')
fi
echo "${FILE_NAME} ${image}" >> ${IMAGE_LIST}
done < "${IMAGES}"
cd ..
sudo chown ${USER} ${IMAGE_DIR}/*
tar -zcvf ${IMAGE_TAR_FILE} ./container-images
rm -rf ${IMAGE_DIR}
echo ""
echo "${IMAGE_TAR_FILE} is created to contain your container images."
echo "Please keep this file and bring it to your offline environment."
}
function register_container_images() {
create_registry=false
REGISTRY_PORT=${REGISTRY_PORT:-"5000"}
if [ -z "${DESTINATION_REGISTRY}" ]; then
echo "DESTINATION_REGISTRY not set, will create local registry"
create_registry=true
DESTINATION_REGISTRY="$(hostname):${REGISTRY_PORT}"
fi
echo "Images will be pushed to ${DESTINATION_REGISTRY}"
if [ ! -f ${IMAGE_TAR_FILE} ]; then
echo "${IMAGE_TAR_FILE} should exist."
exit 1
fi
if [ ! -d ${TEMP_DIR} ]; then
mkdir ${TEMP_DIR}
fi
# To avoid "http: server gave http response to https client" error.
if [ -d /etc/docker/ ]; then
set -e
# Ubuntu18.04, RHEL7/CentOS7
cp ${CURRENT_DIR}/docker-daemon.json ${TEMP_DIR}/docker-daemon.json
sed -i s@"HOSTNAME"@"$(hostname)"@ ${TEMP_DIR}/docker-daemon.json
sudo cp ${TEMP_DIR}/docker-daemon.json /etc/docker/daemon.json
elif [ -d /etc/containers/ ]; then
set -e
# RHEL8/CentOS8
cp ${CURRENT_DIR}/registries.conf ${TEMP_DIR}/registries.conf
sed -i s@"HOSTNAME"@"$(hostname)"@ ${TEMP_DIR}/registries.conf
sudo cp ${TEMP_DIR}/registries.conf /etc/containers/registries.conf
elif [ "$(uname)" == "Darwin" ]; then
echo "This is a Mac, no configuration changes are required"
else
echo "runtime package(docker-ce, podman, nerctl, etc.) should be installed"
exit 1
fi
tar -zxvf ${IMAGE_TAR_FILE}
if [ "${create_registry}" ]; then
sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar
set +e
sudo ${runtime} container inspect registry >/dev/null 2>&1
if [ $? -ne 0 ]; then
sudo ${runtime} run --restart=always -d -p "${REGISTRY_PORT}":"${REGISTRY_PORT}" --name registry registry:latest
fi
set -e
fi
while read -r line; do
file_name=$(echo ${line} | awk '{print $1}')
raw_image=$(echo ${line} | awk '{print $2}')
new_image="${DESTINATION_REGISTRY}/${raw_image}"
load_image=$(sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} | head -n1)
org_image=$(echo "${load_image}" | awk '{print $3}')
# special case for tags containing the digest when using docker or podman as the container runtime
if [ "${org_image}" == "ID:" ]; then
org_image=$(echo "${load_image}" | awk '{print $4}')
fi
image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
if [ -z "${file_name}" ]; then
echo "Failed to get file_name for line ${line}"
exit 1
fi
if [ -z "${raw_image}" ]; then
echo "Failed to get raw_image for line ${line}"
exit 1
fi
if [ -z "${org_image}" ]; then
echo "Failed to get org_image for line ${line}"
exit 1
fi
if [ -z "${image_id}" ]; then
echo "Failed to get image_id for file ${file_name}"
exit 1
fi
sudo ${runtime} load -i ${IMAGE_DIR}/${file_name}
sudo ${runtime} tag ${image_id} ${new_image}
sudo ${runtime} push ${new_image}
done <<< "$(cat ${IMAGE_LIST})"
echo "Succeeded to register container images to local registry."
echo "Please specify \"${DESTINATION_REGISTRY}\" for the following options in your inventry:"
echo "- kube_image_repo"
echo "- gcr_image_repo"
echo "- docker_image_repo"
echo "- quay_image_repo"
}
# get runtime command
if command -v nerdctl 1>/dev/null 2>&1; then
runtime="nerdctl"
elif command -v podman 1>/dev/null 2>&1; then
runtime="podman"
elif command -v docker 1>/dev/null 2>&1; then
runtime="docker"
else
echo "No supported container runtime found"
exit 1
fi
if [ "${OPTION}" == "create" ]; then
create_container_image_tar
elif [ "${OPTION}" == "register" ]; then
register_container_images
else
echo "This script has two features:"
echo "(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE"
echo " environment variable to get images from a file (e.g. temp/images.list after running the"
echo " ./generate_list.sh script)."
echo "(2) Deploy local container registry and register the container images to the registry."
echo ""
echo "Step(1) should be done online site as a preparation, then we bring"
echo "the gotten images to the target offline environment. if images are from"
echo "a private registry, you need to set PRIVATE_REGISTRY environment variable."
echo "Then we will run step(2) for registering the images to local registry, or to an existing"
echo "registry set by the DESTINATION_REGISTRY environment variable. By default, the local registry"
echo "will run on port 5000. This can be changed with the REGISTRY_PORT environment variable"
echo ""
echo "${IMAGE_TAR_FILE} is created to contain your container images."
echo "Please keep this file and bring it to your offline environment."
echo ""
echo "Step(1) can be operated with:"
echo " $ ./manage-offline-container-images.sh create"
echo ""
echo "Step(2) can be operated with:"
echo " $ ./manage-offline-container-images.sh register"
echo ""
echo "Please specify 'create' or 'register'."
echo ""
exit 1
fi
-49
View File
@@ -1,49 +0,0 @@
#!/bin/bash
CURRENT_DIR=$( dirname "$(readlink -f "$0")" )
OFFLINE_FILES_DIR_NAME="offline-files"
OFFLINE_FILES_DIR="${CURRENT_DIR}/${OFFLINE_FILES_DIR_NAME}"
OFFLINE_FILES_ARCHIVE="${CURRENT_DIR}/offline-files.tar.gz"
FILES_LIST=${FILES_LIST:-"${CURRENT_DIR}/temp/files.list"}
NGINX_PORT=8080
# download files
if [ ! -f "${FILES_LIST}" ]; then
echo "${FILES_LIST} should exist, run ./generate_list.sh first."
exit 1
fi
rm -rf "${OFFLINE_FILES_DIR}"
rm "${OFFLINE_FILES_ARCHIVE}"
mkdir "${OFFLINE_FILES_DIR}"
while read -r url; do
if ! wget -x -P "${OFFLINE_FILES_DIR}" "${url}"; then
exit 1
fi
done < "${FILES_LIST}"
tar -czvf "${OFFLINE_FILES_ARCHIVE}" "${OFFLINE_FILES_DIR_NAME}"
[ -n "$NO_HTTP_SERVER" ] && echo "skip to run nginx" && exit 0
# run nginx container server
if command -v nerdctl 1>/dev/null 2>&1; then
runtime="nerdctl"
elif command -v podman 1>/dev/null 2>&1; then
runtime="podman"
elif command -v docker 1>/dev/null 2>&1; then
runtime="docker"
else
echo "No supported container runtime found"
exit 1
fi
sudo "${runtime}" container inspect nginx >/dev/null 2>&1
if [ $? -ne 0 ]; then
sudo "${runtime}" run \
--restart=always -d -p ${NGINX_PORT}:80 \
--volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \
--volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \
--name nginx nginx:alpine
fi
-39
View File
@@ -1,39 +0,0 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
default_type application/octet-stream;
include /etc/nginx/conf.d/*.conf;
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
include /etc/nginx/default.d/*.conf;
location / {
root /usr/share/nginx/html/download;
autoindex on;
autoindex_exact_size off;
autoindex_localtime on;
}
error_page 404 /404.html;
location = /40x.html {
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
}
}
}
-8
View File
@@ -1,8 +0,0 @@
[registries.search]
registries = ['registry.access.redhat.com', 'registry.redhat.io', 'docker.io']
[registries.insecure]
registries = ['HOSTNAME:5000']
[registries.block]
registries = []
-65
View File
@@ -1,65 +0,0 @@
#!/usr/bin/env python3
"""This is a helper script to manage-offline-files.sh.
After running manage-offline-files.sh, you can run upload2artifactory.py
to recursively upload each file to a generic repository in Artifactory.
This script recurses the current working directory and is intended to
be started from 'kubespray/contrib/offline/offline-files'
Environment Variables:
USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'.
TOKEN -- Generate this with 'Set Me Up' in your user.
BASE_URL -- The URL including the repository name.
"""
import os
import urllib.request
import base64
def upload_file(file_path, destination_url, username, token):
"""Helper function to upload a single file"""
try:
with open(file_path, 'rb') as f:
file_data = f.read()
request = urllib.request.Request(destination_url, data=file_data, method='PUT') # NOQA
auth_header = base64.b64encode(f"{username}:{token}".encode()).decode()
request.add_header("Authorization", f"Basic {auth_header}")
with urllib.request.urlopen(request) as response:
if response.status in [200, 201]:
print(f"Success: Uploaded {file_path}")
else:
print(f"Failed: {response.status} {response.read().decode('utf-8')}") # NOQA
except urllib.error.HTTPError as e:
print(f"HTTPError: {e.code} {e.reason} for {file_path}")
except urllib.error.URLError as e:
print(f"URLError: {e.reason} for {file_path}")
except OSError as e:
print(f"OSError: {e.strerror} for {file_path}")
def upload_files(base_url, username, token):
""" Recurse current dir and upload each file using urllib.request """
for root, _, files in os.walk(os.getcwd()):
for file in files:
file_path = os.path.join(root, file)
relative_path = os.path.relpath(file_path, os.getcwd())
destination_url = f"{base_url}/{relative_path}"
print(f"Uploading {file_path} to {destination_url}")
upload_file(file_path, destination_url, username, token)
if __name__ == "__main__":
a_user = os.getenv("USERNAME")
a_token = os.getenv("TOKEN")
a_url = os.getenv("BASE_URL")
if not a_user or not a_token or not a_url:
print(
"Error: Environment variables USERNAME, TOKEN, and BASE_URL must be set." # NOQA
)
exit()
upload_files(a_url, a_user, a_token)
-5
View File
@@ -1,5 +0,0 @@
---
- name: Disable firewalld/ufw
hosts: all
roles:
- { role: prepare }
@@ -1,2 +0,0 @@
---
disable_service_firewall: false
@@ -1,23 +0,0 @@
---
- name: Disable firewalld and ufw
when:
- disable_service_firewall is defined and disable_service_firewall
block:
- name: List services
service_facts:
- name: Disable service firewalld
systemd_service:
name: firewalld
state: stopped
enabled: false
when:
"'firewalld.service' in services and services['firewalld.service'].status != 'not-found'"
- name: Disable service ufw
systemd_service:
name: ufw
state: stopped
enabled: false
when:
"'ufw.service' in services and services['ufw.service'].status != 'not-found'"
-3
View File
@@ -1,3 +0,0 @@
*.tfstate*
.terraform.lock.hcl
.terraform
-124
View File
@@ -1,124 +0,0 @@
# Kubernetes on AWS with Terraform
## Overview
This project will create:
- VPC with Public and Private Subnets in # Availability Zones
- Bastion Hosts and NAT Gateways in the Public Subnet
- A dynamic number of masters, etcd, and worker nodes in the Private Subnet
- even distributed over the # of Availability Zones
- AWS ELB in the Public Subnet for accessing the Kubernetes API from the internet
## Requirements
- Terraform 0.12.0 or newer
## How to Use
- Export the variables for your AWS credentials or edit `credentials.tfvars`:
```commandline
export TF_VAR_AWS_ACCESS_KEY_ID="www"
export TF_VAR_AWS_SECRET_ACCESS_KEY ="xxx"
export TF_VAR_AWS_SSH_KEY_NAME="yyy"
export TF_VAR_AWS_DEFAULT_REGION="zzz"
```
- Update `contrib/terraform/aws/terraform.tfvars` with your data. By default, the Terraform scripts use Ubuntu 18.04 LTS (Bionic) as base image. If you want to change this behaviour, see note "Using other distrib than Ubuntu" below.
- Create an AWS EC2 SSH Key
- Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials
Example:
```commandline
terraform apply -var-file=credentials.tfvars
```
- Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory`
- Ansible will automatically generate an ssh config file for your bastion hosts. To connect to hosts with ssh using bastion host use generated `ssh-bastion.conf`. Ansible automatically detects bastion and changes `ssh_args`
```commandline
ssh -F ./ssh-bastion.conf user@$ip
```
- Once the infrastructure is created, you can run the kubespray playbooks and supply inventory/hosts with the `-i` flag.
Example (this one assumes you are using Ubuntu)
```commandline
ansible-playbook -i ./inventory/hosts ./cluster.yml -e ansible_user=ubuntu -b --become-user=root --flush-cache
```
## Using other distrib than Ubuntu***
To leverage a Linux distribution other than Ubuntu 18.04 (Bionic) LTS for your Terraform configurations, you can adjust the AMI search filters within the 'data "aws_ami" "distro"' block by utilizing variables in your `terraform.tfvars` file. This approach ensures a flexible configuration that adapts to various Linux distributions without directly modifying the core Terraform files.
### Example Usages
- **Debian Jessie**: To configure the usage of Debian Jessie, insert the subsequent lines into your `terraform.tfvars`:
```hcl
ami_name_pattern = "debian-jessie-amd64-hvm-*"
ami_owners = ["379101102735"]
```
- **Ubuntu 16.04**: To utilize Ubuntu 16.04 instead, apply the following configuration in your `terraform.tfvars`:
```hcl
ami_name_pattern = "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-*"
ami_owners = ["099720109477"]
```
- **Centos 7**: For employing Centos 7, incorporate these lines into your `terraform.tfvars`:
```hcl
ami_name_pattern = "dcos-centos7-*"
ami_owners = ["688023202711"]
```
## Connecting to Kubernetes
You can use the following set of commands to get the kubeconfig file from your newly created cluster. Before running the commands, make sure you are in the project's root folder.
```commandline
# Get the controller's IP address.
CONTROLLER_HOST_NAME=$(cat ./inventory/hosts | grep "\[kube_control_plane\]" -A 1 | tail -n 1)
CONTROLLER_IP=$(cat ./inventory/hosts | grep $CONTROLLER_HOST_NAME | grep ansible_host | cut -d'=' -f2)
# Get the hostname of the load balancer.
LB_HOST=$(cat inventory/hosts | grep apiserver_loadbalancer_domain_name | cut -d'"' -f2)
# Get the controller's SSH fingerprint.
ssh-keygen -R $CONTROLLER_IP > /dev/null 2>&1
ssh-keyscan -H $CONTROLLER_IP >> ~/.ssh/known_hosts 2>/dev/null
# Get the kubeconfig from the controller.
mkdir -p ~/.kube
ssh -F ssh-bastion.conf centos@$CONTROLLER_IP "sudo chmod 644 /etc/kubernetes/admin.conf"
scp -F ssh-bastion.conf centos@$CONTROLLER_IP:/etc/kubernetes/admin.conf ~/.kube/config
sed -i "s^server:.*^server: https://$LB_HOST:6443^" ~/.kube/config
kubectl get nodes
```
## Troubleshooting
### Remaining AWS IAM Instance Profile
If the cluster was destroyed without using Terraform it is possible that
the AWS IAM Instance Profiles still remain. To delete them you can use
the `AWS CLI` with the following command:
```commandline
aws iam delete-instance-profile --region <region_name> --instance-profile-name <profile_name>
```
### Ansible Inventory doesn't get created
It could happen that Terraform doesn't create an Ansible Inventory file automatically. If this is the case copy the output after `inventory=` and create a file named `hosts`in the directory `inventory` and paste the inventory into the file.
## Architecture
Pictured is an AWS Infrastructure created with this Terraform project distributed over two Availability Zones.
![AWS Infrastructure with Terraform ](docs/aws_kubespray.png)
@@ -1,179 +0,0 @@
terraform {
required_version = ">= 0.12.0"
}
provider "aws" {
access_key = var.AWS_ACCESS_KEY_ID
secret_key = var.AWS_SECRET_ACCESS_KEY
region = var.AWS_DEFAULT_REGION
}
data "aws_availability_zones" "available" {}
/*
* Calling modules who create the initial AWS VPC / AWS ELB
* and AWS IAM Roles for Kubernetes Deployment
*/
module "aws-vpc" {
source = "./modules/vpc"
aws_cluster_name = var.aws_cluster_name
aws_vpc_cidr_block = var.aws_vpc_cidr_block
aws_avail_zones = data.aws_availability_zones.available.names
aws_cidr_subnets_private = var.aws_cidr_subnets_private
aws_cidr_subnets_public = var.aws_cidr_subnets_public
default_tags = var.default_tags
}
module "aws-nlb" {
source = "./modules/nlb"
aws_cluster_name = var.aws_cluster_name
aws_vpc_id = module.aws-vpc.aws_vpc_id
aws_avail_zones = data.aws_availability_zones.available.names
aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public
aws_nlb_api_port = var.aws_nlb_api_port
k8s_secure_api_port = var.k8s_secure_api_port
default_tags = var.default_tags
}
module "aws-iam" {
source = "./modules/iam"
aws_cluster_name = var.aws_cluster_name
}
/*
* Create Bastion Instances in AWS
*
*/
resource "aws_instance" "bastion-server" {
ami = data.aws_ami.distro.id
instance_type = var.aws_bastion_size
count = var.aws_bastion_num
associate_public_ip_address = true
subnet_id = element(module.aws-vpc.aws_subnet_ids_public, count.index)
vpc_security_group_ids = module.aws-vpc.aws_security_group
key_name = var.AWS_SSH_KEY_NAME
tags = merge(var.default_tags, tomap({
Name = "kubernetes-${var.aws_cluster_name}-bastion-${count.index}"
Cluster = var.aws_cluster_name
Role = "bastion-${var.aws_cluster_name}-${count.index}"
}))
}
/*
* Create K8s Master and worker nodes and etcd instances
*
*/
resource "aws_instance" "k8s-master" {
ami = data.aws_ami.distro.id
instance_type = var.aws_kube_master_size
count = var.aws_kube_master_num
subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)
vpc_security_group_ids = module.aws-vpc.aws_security_group
root_block_device {
volume_size = var.aws_kube_master_disk_size
}
iam_instance_profile = module.aws-iam.kube_control_plane-profile
key_name = var.AWS_SSH_KEY_NAME
tags = merge(var.default_tags, tomap({
Name = "kubernetes-${var.aws_cluster_name}-master${count.index}"
"kubernetes.io/cluster/${var.aws_cluster_name}" = "member"
Role = "master"
}))
}
resource "aws_lb_target_group_attachment" "tg-attach_master_nodes" {
count = var.aws_kube_master_num
target_group_arn = module.aws-nlb.aws_nlb_api_tg_arn
target_id = element(aws_instance.k8s-master.*.private_ip, count.index)
}
resource "aws_instance" "k8s-etcd" {
ami = data.aws_ami.distro.id
instance_type = var.aws_etcd_size
count = var.aws_etcd_num
subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)
vpc_security_group_ids = module.aws-vpc.aws_security_group
root_block_device {
volume_size = var.aws_etcd_disk_size
}
key_name = var.AWS_SSH_KEY_NAME
tags = merge(var.default_tags, tomap({
Name = "kubernetes-${var.aws_cluster_name}-etcd${count.index}"
"kubernetes.io/cluster/${var.aws_cluster_name}" = "member"
Role = "etcd"
}))
}
resource "aws_instance" "k8s-worker" {
ami = data.aws_ami.distro.id
instance_type = var.aws_kube_worker_size
count = var.aws_kube_worker_num
subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)
vpc_security_group_ids = module.aws-vpc.aws_security_group
root_block_device {
volume_size = var.aws_kube_worker_disk_size
}
iam_instance_profile = module.aws-iam.kube-worker-profile
key_name = var.AWS_SSH_KEY_NAME
tags = merge(var.default_tags, tomap({
Name = "kubernetes-${var.aws_cluster_name}-worker${count.index}"
"kubernetes.io/cluster/${var.aws_cluster_name}" = "member"
Role = "worker"
}))
}
/*
* Create Kubespray Inventory File
*
*/
data "template_file" "inventory" {
template = file("${path.module}/templates/inventory.tpl")
vars = {
public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip))
connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip))
connection_strings_node = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip))
list_master = join("\n", aws_instance.k8s-master.*.private_dns)
list_node = join("\n", aws_instance.k8s-worker.*.private_dns)
connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
list_etcd = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))
nlb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-nlb.aws_nlb_api_fqdn}\""
}
}
resource "null_resource" "inventories" {
provisioner "local-exec" {
command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
}
triggers = {
template = data.template_file.inventory.rendered
}
}
@@ -1,8 +0,0 @@
#AWS Access Key
AWS_ACCESS_KEY_ID = ""
#AWS Secret Key
AWS_SECRET_ACCESS_KEY = ""
#EC2 SSH Key Name
AWS_SSH_KEY_NAME = ""
#AWS Region
AWS_DEFAULT_REGION = "eu-central-1"
Binary file not shown.

Before

Width:  |  Height:  |  Size: 114 KiB

-141
View File
@@ -1,141 +0,0 @@
#Add AWS Roles for Kubernetes
resource "aws_iam_role" "kube_control_plane" {
name = "kubernetes-${var.aws_cluster_name}-master"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"Service": "ec2.amazonaws.com"
}
}
]
}
EOF
}
resource "aws_iam_role" "kube-worker" {
name = "kubernetes-${var.aws_cluster_name}-node"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"Service": "ec2.amazonaws.com"
}
}
]
}
EOF
}
#Add AWS Policies for Kubernetes
resource "aws_iam_role_policy" "kube_control_plane" {
name = "kubernetes-${var.aws_cluster_name}-master"
role = aws_iam_role.kube_control_plane.id
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["ec2:*"],
"Resource": ["*"]
},
{
"Effect": "Allow",
"Action": ["elasticloadbalancing:*"],
"Resource": ["*"]
},
{
"Effect": "Allow",
"Action": ["route53:*"],
"Resource": ["*"]
},
{
"Effect": "Allow",
"Action": "s3:*",
"Resource": [
"arn:aws:s3:::kubernetes-*"
]
}
]
}
EOF
}
resource "aws_iam_role_policy" "kube-worker" {
name = "kubernetes-${var.aws_cluster_name}-node"
role = aws_iam_role.kube-worker.id
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "s3:*",
"Resource": [
"arn:aws:s3:::kubernetes-*"
]
},
{
"Effect": "Allow",
"Action": "ec2:Describe*",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "ec2:AttachVolume",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "ec2:DetachVolume",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": ["route53:*"],
"Resource": ["*"]
},
{
"Effect": "Allow",
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:GetRepositoryPolicy",
"ecr:DescribeRepositories",
"ecr:ListImages",
"ecr:BatchGetImage"
],
"Resource": "*"
}
]
}
EOF
}
#Create AWS Instance Profiles
resource "aws_iam_instance_profile" "kube_control_plane" {
name = "kube_${var.aws_cluster_name}_master_profile"
role = aws_iam_role.kube_control_plane.name
}
resource "aws_iam_instance_profile" "kube-worker" {
name = "kube_${var.aws_cluster_name}_node_profile"
role = aws_iam_role.kube-worker.name
}
@@ -1,7 +0,0 @@
output "kube_control_plane-profile" {
value = aws_iam_instance_profile.kube_control_plane.name
}
output "kube-worker-profile" {
value = aws_iam_instance_profile.kube-worker.name
}

Some files were not shown because too many files have changed in this diff Show More