disable bgp for master

Separate apps deploy from cluster deploy

.gitignore

@@ -1,2 +0,0 @@
-ssh
-nodes

.gitmodules

@@ -0,0 +1,43 @@
+[submodule "roles/apps/k8s-kube-ui"]
+    path = roles/apps/k8s-kube-ui
+    url = https://github.com/ansibl8s/k8s-kube-ui.git
+    branch = v1.0
+[submodule "roles/apps/k8s-kubedns"]
+    path = roles/apps/k8s-kubedns
+    url = https://github.com/ansibl8s/k8s-kubedns.git
+    branch = v1.0
+[submodule "roles/apps/k8s-common"]
+    path = roles/apps/k8s-common
+    url = https://github.com/ansibl8s/k8s-common.git
+    branch = v1.0
+[submodule "roles/apps/k8s-redis"]
+    path = roles/apps/k8s-redis
+    url = https://github.com/ansibl8s/k8s-redis.git
+    branch = v1.0
+[submodule "roles/apps/k8s-elasticsearch"]
+    path = roles/apps/k8s-elasticsearch
+    url = https://github.com/ansibl8s/k8s-elasticsearch.git
+[submodule "roles/apps/k8s-fabric8"]
+    path = roles/apps/k8s-fabric8
+    url = https://github.com/ansibl8s/k8s-fabric8.git
+    branch = v1.0
+[submodule "roles/apps/k8s-memcached"]
+    path = roles/apps/k8s-memcached
+    url = https://github.com/ansibl8s/k8s-memcached.git
+    branch = v1.0
+[submodule "roles/apps/k8s-postgres"]
+    path = roles/apps/k8s-postgres
+    url = https://github.com/ansibl8s/k8s-postgres.git
+    branch = v1.0
+[submodule "roles/apps/k8s-kubedash"]
+    path = roles/apps/k8s-kubedash
+    url = https://github.com/ansibl8s/k8s-kubedash.git
+[submodule "roles/apps/k8s-heapster"]
+    path = roles/apps/k8s-heapster
+    url = https://github.com/ansibl8s/k8s-heapster.git
+[submodule "roles/apps/k8s-influxdb"]
+    path = roles/apps/k8s-influxdb
+    url = https://github.com/ansibl8s/k8s-influxdb.git
+[submodule "roles/apps/k8s-kube-logstash"]
+	path = roles/apps/k8s-kube-logstash
+	url = https://github.com/ansibl8s/k8s-kube-logstash.git

README.md

@@ -1,161 +1,240 @@
-vagrant-k8s
+kubernetes-ansible
-===========
+========
-Scripts to create libvirt lab with vagrant and prepare some stuff for `k8s` deployment with `kargo`.
+
+Install and configure a kubernetes cluster including network plugin and optionnal addons.
+Based on [CiscoCloud](https://github.com/CiscoCloud/kubernetes-ansible) work.
+
+### Requirements
+Tested on **Debian Jessie** and **Ubuntu** (14.10, 15.04, 15.10).
+The target servers must have access to the Internet in order to pull docker imaqes.
+The firewalls are not managed, you'll need to implement your own rules the way you used to.
+
+Ansible v1.9.x
+
+### Components
+* [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.2
+* [etcd](https://github.com/coreos/etcd/releases) v2.2.2
+* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.11.0
+* [flanneld](https://github.com/coreos/flannel/releases) v0.5.5
+* [docker](https://www.docker.com/) v1.8.3
 
 
-Requirements
+Ansible
-------------
+-------------------------
+### Download binaries
+A role allows to download required binaries. They will be stored in a directory defined by the variable
+**'local_release_dir'** (by default /tmp).
+Please ensure that you have enough disk space there (about **1G**).
 
-* `libvirt`
+**Note**: Whenever you'll need to change the version of a software, you'll have to erase the content of this directory.
-* `vagrant`
-* `vagrant-libvirt` plugin (`vagrant plugin install vagrant-libvirt`)
-* `$USER` should be able to connect to libvirt (test with `virsh list --all`)
 
-Vargant lab preparation
------------------------
 
-* Change default IP pool for vagrant networks if you want:
+### Variables
+The main variables to change are located in the directory ```environments/[env_name]/group_vars/k8s-cluster.yml```.
 
-```bash
+### Inventory
-export VAGRANT_POOL="10.100.0.0/16"
+Below is an example of an inventory.
+Note : The bgp vars local_as and peers are not mandatory if the var **'peer_with_router'** is set to false
+By default this variable is set to false and therefore all the nodes are configure in **'node-mesh'** mode.
+In node-mesh mode the nodes peers with all the nodes in order to exchange routes.
+
+```
+[downloader]
+10.99.0.26
+
+[kube-master]
+10.99.0.26
+
+[etcd]
+10.99.0.26
+
+[kube-node]
+10.99.0.4
+10.99.0.5
+10.99.0.36
+10.99.0.37
+
+[paris]
+10.99.0.26
+10.99.0.4 local_as=xxxxxxxx
+10.99.0.5 local_as=xxxxxxxx
+
+[usa]
+10.99.0.36 local_as=xxxxxxxx
+10.99.0.37 local_as=xxxxxxxx
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
+[paris:vars]
+peers=[{"router_id": "10.99.0.2", "as": "65xxx"}, {"router_id": "10.99.0.3", "as": "65xxx"}]
+
+[usa:vars]
+peers=[{"router_id": "10.99.0.34", "as": "65xxx"}, {"router_id": "10.99.0.35", "as": "65xxx"}]
 ```
 
-* Clone this repo
+### Playbook
+```
+---
+- hosts: downloader
+  sudo: no
+  roles:
+    - { role: download, tags: download }
 
-```bash
+- hosts: k8s-cluster
-git clone https://github.com/adidenko/vagrant-k8s
+  roles:
-cd vagrant-k8s
+    - { role: etcd, tags: etcd }
+    - { role: docker, tags: docker }
+    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
+    - { role: dnsmasq, tags: dnsmasq }
+
+- hosts: kube-master
+  roles:
+    - { role: kubernetes/master, tags: master }
+
+- hosts: kube-node
+  roles:
+    - { role: kubernetes/node, tags: node }
 ```
 
-* Prepare the virtual lab:
+### Run
-
+It is possible to define variables for different environments.
-```bash
+For instance, in order to deploy the cluster on 'dev' environment run the following command.
-vagrant up
+```
+ansible-playbook -i environments/dev/inventory cluster.yml -u root
 ```
 
-Deployment on a lab
+Kubernetes
--------------------
+-------------------------
 
-* Login to master node and sudo to root:
+### Network Overlay
+You can choose between 2 network plugins. Only one must be chosen.
 
-```bash
+* **flannel**: gre/vxlan (layer 2) networking. ([official docs]('https://github.com/coreos/flannel'))
-vagrant ssh $USER-k8s-00
+
-sudo su -
+* **calico**: bgp (layer 3) networking. ([official docs]('http://docs.projectcalico.org/en/0.13/'))
+
+The choice is defined with the variable '**kube_network_plugin**'
+
+### Expose a service
+There are several loadbalancing solutions.
+The ones i found suitable for kubernetes are [Vulcand]('http://vulcand.io/') and [Haproxy]('http://www.haproxy.org/')
+
+My cluster is working with haproxy and kubernetes services are configured with the loadbalancing type '**nodePort**'.
+eg: each node opens the same tcp port and forwards the traffic to the target pod wherever it is located.
+
+Then Haproxy can be configured to request kubernetes's api in order to loadbalance on the proper tcp port on the nodes.
+
+Please refer to the proper kubernetes documentation on [Services]('https://github.com/kubernetes/kubernetes/blob/release-1.0/docs/user-guide/services.md')
+
+### Check cluster status
+
+#### Kubernetes components
+Master processes : kube-apiserver, kube-scheduler, kube-controller, kube-proxy
+Nodes processes : kubelet, kube-proxy, [calico-node|flanneld]
+
+* Check the status of the processes
+```
+systemctl status [process_name]
 ```
 
-* Clone this repo
+* Check the logs
-
+```
-```bash
+journalctl -ae -u [process_name]
-git clone https://github.com/adidenko/vagrant-k8s ~/mcp
 ```
 
-* Install required software and pull needed repos:
+* Check the NAT rules
-
+```
-```bash
+iptables -nLv -t nat
-cd ~/mcp
-./bootstrap-master.sh
 ```
 
-* Check `nodes` list and make sure you have SSH access to them
 
-```bash
+### Available apps, installation procedure
-cd ~/mcp
+
-cat nodes
+There are two ways of installing new apps
-ansible all -m ping -i nodes_to_inv.py
+
+#### Ansible galaxy
+
+Additionnal apps can be installed with ```ansible-galaxy```.
+
+ou'll need to edit the file '*requirements.yml*' in order to chose needed apps.
+The list of available apps are available [there](https://github.com/ansibl8s)
+
+For instance it is **strongly recommanded** to install a dns server which resolves kubernetes service names.
+In order to use this role you'll need the following entries in the file '*requirements.yml*' 
+Please refer to the [k8s-kubdns readme](https://github.com/ansibl8s/k8s-kubedns) for additionnal info.
+```
+- src: https://github.com/ansibl8s/k8s-common.git
+  path: roles/apps
+  # version: v1.0
+
+- src: https://github.com/ansibl8s/k8s-kubedns.git
+  path: roles/apps
+  # version: v1.0
+```
+**Note**: the role common is required by all the apps and provides the tasks and libraries needed.
+
+And empty the apps directory
+```
+rm -rf roles/apps/*
 ```
 
-* Deploy k8s using kargo playbooks
+Then download the roles with ansible-galaxy
-
+```
-```bash
+ansible-galaxy install -r requirements.yml
-cd ~/mcp
-./deploy-k8s.kargo.sh
 ```
 
-* Deploy OpenStack CCP:
+#### Git submodules
+Alternatively the roles can be installed as git submodules.
+That way is easier if you want to do some changes and commit them.
 
-```bash
+You can list available submodules with the following command:
-cd ~/mcp
+```
-# Build CCP images
+grep path .gitmodules | sed 's/.*= //'
-ansible-playbook -i nodes_to_inv.py playbooks/ccp-build.yaml
-# Deploy CCP
-ansible-playbook -i nodes_to_inv.py playbooks/ccp-deploy.yaml
 ```
 
-* Wait for CCP deployment to complete
+In order to install the dns addon you'll need to follow these steps
-
+```
-```bash
+git submodule init roles/apps/k8s-common roles/apps/k8s-kubedns
-# On k8s master node
+git submodule update
-# Check CCP pods, all should become running
-kubectl --namespace=openstack get pods -o wide
-
-# Check CCP jobs status, wait until all complete
-kubectl --namespace=openstack get jobs
 ```
 
-* Check Horizon:
+Finally update the playbook ```apps.yml``` with the chosen roles, and run it
-
+```
-```bash
+...
-# On k8s master node check nodePort of Horizon service
+- hosts: kube-master
-HORIZON_PORT=$(kubectl --namespace=openstack get svc/horizon -o go-template='{{(index .spec.ports 0).nodePort}}')
+  roles:
-echo $HORIZON_PORT
+    - { role: apps/k8s-kubedns, tags: ['kubedns', 'apps']  }
-
+...
-# Access Horizon via nodePort
-curl -i -s $ANY_K8S_NODE_IP:$HORIZON_PORT
 ```
 
-Working with kubernetes
+```
------------------------
+ansible-playbook -i environments/dev/inventory apps.yml -u root
-
-* Login to one of your kube-master nodes and run:
-
-```bash
-# List images in registry
-curl -s 127.0.0.1:31500/v2/_catalog | python -mjson.tool
-
-# Check CCP jobs status
-kubectl --namespace=openstack get jobs
-
-# Check CCP pods
-kubectl --namespace=openstack get pods -o wide
 ```
 
-* Troubleshooting
 
-```bash
+#### Calico networking
-# Get logs from pod
+Check if the calico-node container is running
-kubectl --namespace=openstack logs $POD_NAME
+```
-
+docker ps | grep calico
-# Exec command from pod
-kubectl --namespace=openstack exec $POD_NAME -- cat /etc/resolv.conf
-kubectl --namespace=openstack exec $POD_NAME -- curl http://etcd-client:2379/health
-
-# Run a container
-docker run -t -i 127.0.0.1:31500/mcp/neutron-dhcp-agent /bin/bash
 ```
 
-* Network checker
+The **calicoctl** command allows to check the status of the network workloads.
-
+* Check the status of Calico nodes
-```bash
+```
-cd ~/mcp
+calicoctl status
-./deploy-netchecker.sh
-# or in openstack namespace
-./deploy-netchecker.sh openstack
 ```
 
-* CCP
+* Show the configured network subnet for containers
-
-```bash
-# Run a bash in one of containers
-docker run -t -i 127.0.0.1:31500/mcp/nova-base /bin/bash
-
-# Inside container export credentials
-export OS_USERNAME=admin
-export OS_PASSWORD=password
-export OS_TENANT_NAME=admin
-export OS_REGION_NAME=RegionOne
-export OS_AUTH_URL=http://keystone:35357
-
-# Run CLI commands
-openstack service list
-neutron agent-list
 ```
+calicoctl pool show
+```
+
+* Show the workloads (ip addresses of containers and their located)
+```
+calicoctl endpoint show --detail
+```
+#### Flannel networking
+
+Congrats ! now you can walk through [kubernetes basics](http://kubernetes.io/v1.1/basicstutorials.html)

Vagrantfile

@@ -1,115 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-pool = ENV["VAGRANT_POOL"] || "10.250.0.0/16"
-
-ENV["VAGRANT_DEFAULT_PROVIDER"] = "libvirt"
-prefix = pool.gsub(/\.\d+\.\d+\/16$/, "")
-
-$num_instances = 4
-$vm_memory = 6144
-$vm_cpus = 2
-$master_memory = 1024
-$master_cpus = 1
-
-$user = ENV["USER"]
-$public_subnet = prefix.to_s + ".0"
-$private_subnet = prefix.to_s + ".1"
-$mgmt_cidr = prefix.to_s + ".2.0/24"
-$neutron_subnet = "172.30.250"
-
-$instance_name_prefix = "#{$user}-k8s"
-
-# Boxes with libvirt provider support:
-#$box = "yk0/ubuntu-xenial" #900M
-#$box = "centos/7"
-#$box = "nrclark/xenial64-minimal-libvirt"
-$box = "peru/ubuntu-16.04-server-amd64"
-
-# Create SSH keys for future lab
-system 'bash vagrant-scripts/ssh-keygen.sh'
-
-# Create nodes list for future kargo deployment
-nodes=""
-(1..$num_instances-1).each do |i|
-  ip = "#{$private_subnet}.#{i+10}"
-  nodes = "#{nodes}#{ip}\n"
-end
-File.open("nodes", 'w') { |file| file.write(nodes) }
-
-# Create the lab
-Vagrant.configure("2") do |config|
-  (0..$num_instances-1).each do |i|
-    # First node would be master node
-    master = i == 0
-
-    config.ssh.insert_key = false
-    vm_name = "%s-%02d" % [$instance_name_prefix, i]
-
-    config.vm.define vm_name do |test_vm|
-      test_vm.vm.box = $box
-      test_vm.vm.hostname = vm_name
-
-      # Libvirt provider settings
-      test_vm.vm.provider :libvirt do |domain|
-        domain.uri = "qemu+unix:///system"
-        if master
-          domain.memory = $master_memory
-          domain.cpus = $master_cpus
-        else
-          domain.memory = $vm_memory
-          domain.cpus = $vm_cpus
-        end
-        domain.driver = "kvm"
-        domain.host = "localhost"
-        domain.connect_via_ssh = false
-        domain.username = $user
-        domain.storage_pool_name = "default"
-        domain.nic_model_type = "e1000"
-        domain.management_network_name = "#{$instance_name_prefix}-mgmt-net"
-        domain.management_network_address = $mgmt_cidr
-        domain.nested = true
-        domain.cpu_mode = "host-passthrough"
-        domain.volume_cache = "unsafe"
-        domain.disk_bus = "virtio"
-        # DISABLED: switched to new box which has 100G / partition
-        #domain.storage :file, :type => 'qcow2', :bus => 'virtio', :size => '20G', :device => 'vdb'
-      end
-
-      # Networks and interfaces
-      ip = "#{$private_subnet}.#{i+10}"
-      pub_ip = "#{$public_subnet}.#{i+10}"
-      # "public" network with nat forwarding
-      test_vm.vm.network :private_network,
-        :ip => pub_ip,
-        :model_type => "e1000",
-        :libvirt__network_name => "#{$instance_name_prefix}-public",
-        :libvirt__dhcp_enabled => false,
-        :libvirt__forward_mode => "nat"
-      # "private" isolated network
-      test_vm.vm.network :private_network,
-        :ip => ip,
-        :model_type => "e1000",
-        :libvirt__network_name => "#{$instance_name_prefix}-private",
-        :libvirt__dhcp_enabled => false,
-        :libvirt__forward_mode => "none"
-      # "neutron" isolated network
-      test_vm.vm.network :private_network,
-        :ip => "#{$neutron_subnet}.#{i+10}",
-        :model_type => "e1000",
-        :libvirt__network_name => "#{$instance_name_prefix}-neutron",
-        :libvirt__dhcp_enabled => false,
-        :libvirt__forward_mode => "none"
-
-      # Provisioning
-      config.vm.provision "file", source: "ssh", destination: "~/ssh"
-      if master
-        config.vm.provision "nodes", type: "file", source: "nodes", destination: "/var/tmp/nodes"
-        config.vm.provision "bootstrap", type: "shell", path: "vagrant-scripts/provision-master.sh"
-      else
-        config.vm.provision "bootstrap", type: "shell", path: "vagrant-scripts/provision-node.sh"
-      end
-
-    end
-  end
-end

apps.yml

@@ -0,0 +1,23 @@
+---
+- hosts: kube-master
+  roles:
+    # System
+    - { role: apps/k8s-kubedns, tags: 'kubedns' }
+
+    # Databases
+    - { role: apps/k8s-postgres, tags: 'postgres' }
+    - { role: apps/k8s-elasticsearch, tags: 'es' }
+    - { role: apps/k8s-memcached, tags: 'es' }
+    - { role: apps/k8s-redis, tags: 'es' }
+
+    # Monitoring
+    - { role: apps/k8s-influxdb, tags: 'influxdb'}
+    - { role: apps/k8s-heapster, tags: 'heapster'}
+    - { role: apps/k8s-kubedash, tags: 'kubedash'}
+
+    # logging
+    - { role: apps/k8s-kube-logstash, tags: 'kube-logstash'}
+
+    # Console
+    - { role: apps/k8s-fabric8, tags: 'fabric8' }
+    - { role: apps/k8s-kube-ui, tags: 'kube-ui' }

bak/deploy-ccp.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -e
-
-INVENTORY="nodes_to_inv.py"
-
-echo "Createing repository and CCP images, it may take a while..."
-ansible-playbook -i $INVENTORY playbooks/ccp-build.yaml
-
-echo "Deploying up OpenStack CCP..."
-ansible-playbook -i $INVENTORY playbooks/ccp-deploy.yaml

bootstrap-master.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# Packages
-apt-get --yes update
-apt-get --yes upgrade
-apt-get --yes install git screen vim telnet tcpdump python-setuptools gcc python-dev python-pip libssl-dev libffi-dev software-properties-common curl python-netaddr
-
-# Get ansible-2.1+, vanilla ubuntu-16.04 ansible (2.0.0.2) is broken due to https://github.com/ansible/ansible/issues/13876
-ansible --version || (
-  apt-add-repository -y ppa:ansible/ansible
-  apt-get update
-  apt-get install -y ansible
-)
-
-# Copy/create nodes list
-test -f ./nodes || cp /var/tmp/nodes ./nodes
-
-# Either pull or copy microservices repos
-cp -a /var/tmp/microservices* ./ccp/ || touch /var/tmp/ccp-download
-
-# Pull kargo
-git clone https://github.com/kubespray/kargo ~/kargo

ccp/.gitignore

@@ -1,2 +0,0 @@
-microservices-repos
-microservices

ccp/ccp.conf

@@ -1,16 +0,0 @@
-[DEFAULT]
-deploy_config = /root/ccp/deploy-config.yaml
-
-[builder]
-push = True
-
-[registry]
-address = "127.0.0.1:31500"
-
-[kubernetes]
-namespace = "openstack"
-
-[repositories]
-skip_empty = True
-protocol = https
-port = 443

ccp/deploy-config.yaml

@@ -1,6 +0,0 @@
-configs:
-  public_interface: "eth1"
-  private_interface: "eth2"
-  neutron_external_interface: "eth3"
-  neutron_logging_debug: "true"
-  neutron_plugin_agent: "openvswitch"

ccp/label-nodes.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# FIXME: hardcoded roles
-declare -A nodes
-nodes=( \
-["node1"]="openstack-controller=true"
-["node2"]="openstack-compute=true"
-["node3"]="openstack-compute=true"
-)
-
-label_nodes() {
-  all_label='openstack-compute-controller=true'
-  for i in "${!nodes[@]}"
-  do
-    node=$i
-    label=${nodes[$i]}
-    kubectl get nodes $node --show-labels | grep -q "$label" || kubectl label nodes $node $label
-    kubectl get nodes $node --show-labels | grep -q "$all_label" || kubectl label nodes $node $all_label
-  done
-}
-
-label_nodes
-

ccp/registry_pod.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: registry
-  labels:
-    app: registry
-spec:
-  containers:
-    - name: registry
-      image: registry:2
-      env:
-      imagePullPolicy: Always
-      ports:
-        - containerPort: 5000
-          hostPort: 5000
-

ccp/registry_svc.yaml

@@ -1,15 +0,0 @@
-kind: "Service"
-apiVersion: "v1"
-metadata:
-  name: "registry"
-spec:
-  selector:
-    app: "registry"
-  ports:
-    -
-      protocol: "TCP"
-      port: 5000
-      targetPort: 5000
-      nodePort: 31500
-  type: "NodePort"
-

cluster.yml

@@ -0,0 +1,20 @@
+---
+- hosts: downloader
+  sudo: no
+  roles:
+    - { role: download, tags: download }
+
+- hosts: k8s-cluster
+  roles:
+    - { role: etcd, tags: etcd }
+    - { role: docker, tags: docker }
+    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
+    - { role: dnsmasq, tags: dnsmasq }
+
+- hosts: kube-master
+  roles:
+    - { role: kubernetes/master, tags: master }
+
+- hosts: kube-node
+  roles:
+    - { role: kubernetes/node, tags: node }

custom.yaml

@@ -1,13 +0,0 @@
-# Kubernetes version
-kube_version: "v1.2.4"
-# Switch network to calico
-kube_network_plugin: "calico"
-# Kube-proxy should be iptables for calico
-kube_proxy_mode: "iptables"
-# Use non-tmpfs tmp dir
-local_release_dir: "/var/tmp/releases"
-# Upstream DNS servers with mirantis.net
-upstream_dns_servers:
-  - 8.8.8.8
-  - 8.8.4.4
-  - /mirantis.net/172.18.32.6

deploy-k8s.kargo.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-INVENTORY="nodes_to_inv.py"
-
-echo "Installing requirements on nodes..."
-ansible-playbook -i $INVENTORY playbooks/bootstrap-nodes.yaml
-
-echo "Running deployment..."
-ansible-playbook -i $INVENTORY /root/kargo/cluster.yml -e @custom.yaml
-deploy_res=$?
-
-if [ "$deploy_res" -eq "0" ]; then
-  echo "Setting up kubedns..."
-  ansible-playbook -i $INVENTORY playbooks/kubedns.yaml
-  echo "Setting up kubedashboard..."
-  ansible-playbook -i $INVENTORY playbooks/kubedashboard.yaml
-  echo "Setting up ip route work-around for DNS clusterIP availability..."
-  ansible-playbook -i $INVENTORY playbooks/ipro_for_cluster_ips.yaml
-fi

deploy-netchecker.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-if [ -n "$1" ] ; then
-  NS="--namespace=$1"
-fi
-
-kubectl get nodes || exit 1
-
-echo "Installing netchecker server"
-git clone https://github.com/adidenko/netchecker-server
-pushd netchecker-server
-  pushd docker
-    docker build -t 127.0.0.1:31500/netchecker/server:latest .
-    docker push 127.0.0.1:31500/netchecker/server:latest
-  popd
-  kubectl create -f netchecker-server_pod.yaml $NS
-  kubectl create -f netchecker-server_svc.yaml $NS
-popd
-
-echo "Installing netchecker agents"
-git clone https://github.com/adidenko/netchecker-agent
-pushd netchecker-agent
-  pushd docker
-    docker build -t 127.0.0.1:31500/netchecker/agent:latest .
-    docker push 127.0.0.1:31500/netchecker/agent:latest
-  popd
-  kubectl get nodes | grep Ready | awk '{print $1}' | xargs -I {} kubectl label nodes {} netchecker=agent
-  NUMNODES=`kubectl get nodes --show-labels | grep Ready | grep netchecker=agent | wc -l`
-  sed -e "s/replicas:.*/replicas: $NUMNODES/g" -i netchecker-agent_rc.yaml
-  kubectl create -f netchecker-agent_rc.yaml $NS
-popd
-
-echo "DONE"
-echo
-echo "use the following command to check agents:"
-echo "curl -s -X GET 'http://localhost:31081/api/v1/agents/' | python -mjson.tool"

environments/production/group_vars/all.yml

@@ -0,0 +1,6 @@
+ # Directory where the binaries will be installed
+bin_dir: /usr/local/bin
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+local_release_dir: "/tmp/releases"

environments/production/group_vars/k8s-cluster.yml

@@ -0,0 +1,60 @@
+# Users to create for basic auth in Kubernetes API via HTTP
+# kube_users:
+#   kube:
+#     pass: changeme
+#     role: admin
+#   root:
+#     pass: changeme
+#     role: admin
+
+# Kubernetes cluster name, also will be used as DNS domain
+# cluster_name: cluster.local
+
+# set this variable to calico if needed. keep it empty if flannel is used
+# kube_network_plugin: calico
+
+# Kubernetes internal network for services, unused block of space.
+# kube_service_addresses: 10.233.0.0/18
+
+# internal network. When used, it will assign IP
+# addresses from this range to individual pods.
+# This network must be unused in your network infrastructure!
+# kube_pods_subnet: 10.233.64.0/18
+
+# internal network total size (optional). This is the prefix of the
+# entire network. Must be unused in your environment.
+# kube_network_prefix: 18
+
+# internal network node size allocation (optional). This is the size allocated
+# to each node on your network.  With these defaults you should have
+# room for 4096 nodes with 254 pods per node.
+# kube_network_node_prefix: 24
+
+# With calico it is possible to distributed routes with border routers of the datacenter.
+# peer_with_router: false
+# Warning : enabling router peering will disable calico's default behavior ('node mesh').
+# The subnets of each nodes will be distributed by the datacenter router
+
+# The port the API Server will be listening on.
+# kube_master_port: 443 # (https)
+# kube_master_insecure_port: 8080 # (http)
+
+# Internal DNS configuration.
+# Kubernetes can create and mainatain its own DNS server to resolve service names
+# into appropriate IP addresses. It's highly advisable to run such DNS server,
+# as it greatly simplifies configuration of your applications - you can use
+# service names instead of magic environment variables.
+# You still must manually configure all your containers to use this DNS server,
+# Kubernetes won't do this for you (yet).
+
+# Upstream dns servers used by dnsmasq
+# upstream_dns_servers:
+#   - 8.8.8.8
+#   - 4.4.8.8
+#
+# # Use dns server : https://github.com/ansibl8s/k8s-skydns/blob/master/skydns-README.md
+# dns_setup: true
+# dns_domain: "{{ cluster_name }}"
+#
+# # Ip address of the kubernetes dns service
+# dns_server: 10.233.0.10

examples/kubernetes/ccp/README.md

@@ -1,25 +0,0 @@
-CCP examples
-============
-Some examples for Openstack CCP.
-
-Expose Horizon
-==============
-
-* Get nodePort of Horizon service:
-```bash
-echo $(kubectl --namespace=openstack get svc/horizon -o go-template='{{(index .spec.ports 0).nodePort}}')
-```
-
-* NAT on your router/jump-box to any k8s minion public IP and nodePort to provide external access:
-```bash
-iptables -t nat -I PREROUTING -p tcp --dport 8080 -j DNAT --to-destination 10.210.0.12:32643
-iptables -t nat -I POSTROUTING -d 10.210.0.12 ! -s 10.210.0.0/24 -j MASQUERADE
-iptables -I FORWARD -d 10.210.0.12 -j ACCEPT
-```
-
-Where `10.210.0.12` is IP of one of your k8s minions and `32643` is nodePort of Horizon service.
-
-* You can do the same for novnc:
-```bash
-echo $(kubectl --namespace=openstack get svc/nova-novncproxy -o go-template='{{(index .spec.ports 0).nodePort}}')
-```

examples/kubernetes/ccp/run_demo.sh

@@ -1,36 +0,0 @@
-# This script should be executed inside k8s:
-# docker run -t -i 127.0.0.1:31500/mcp/nova-base /bin/bash
-
-export OS_USERNAME=admin
-export OS_PASSWORD=password
-export OS_TENANT_NAME=admin
-export OS_REGION_NAME=RegionOne
-export OS_AUTH_URL=http://keystone:35357
-
-# Key
-nova keypair-add test > test.pem
-chmod 600 test.pem
-
-# Flavor
-nova flavor-create demo --is-public true auto 128 2 1
-
-# Image
-curl -O http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img
-glance image-create --name cirros --disk-format qcow2 --container-format bare --file cirros-0.3.4-x86_64-disk.img
-
-# Aggregates
-node2=`openstack hypervisor list | grep -o '[a-z]\+-k8s-02'`
-node3=`openstack hypervisor list | grep -o '[a-z]\+-k8s-03'`
-nova aggregate-create n2 n2
-nova aggregate-add-host n2 $node2
-nova aggregate-create n3 n3
-nova aggregate-add-host n3 $node3
-
-# Network
-neutron net-create net1 --provider:network-type vxlan
-neutron subnet-create net1 172.20.0.0/24 --name subnet1
-
-# Instances
-net_id=`neutron net-list | grep net1 | awk '{print $2}'`
-nova boot ti02 --image cirros --flavor demo --nic net-id=$net_id --key-name test --availability-zone n2
-nova boot ti03 --image cirros --flavor demo --nic net-id=$net_id --key-name test --availability-zone n3

examples/kubernetes/expose-services/README.md

@@ -1,45 +0,0 @@
-Examples how to expose k8s services
-===================================
-
-Exposing dashboard via frontend and externalIPs
------------------------------------------------
-
-* Edit `kubernetes-dashboard.yaml` and update `externalIPs` to the list of external IPs of your k8s minions
-
-* Run:
-
-```bash
-kubectl create -f kubernetes-dashboard.yaml --namespace=kube-system
-```
-
-* Access:
-
-```bash
-curl $ANY_MINION_EXTERNAL_IP:9090
-```
-
-Exposing dashboard via nodePort
--------------------------------
-
-* Get nodePort of the service:
-
-```bash
-echo $(kubectl --namespace=kube-system get svc/kubernetes-dashboard -o go-template='{{(index .spec.ports 0).nodePort}}')
-```
-
-* NAT on your router/jump-box to any k8s minion public IP and nodePort to provide external access:
-
-```bash
-iptables -t nat -I PREROUTING -p tcp --dport 9090 -j DNAT --to-destination 10.210.0.12:32005
-iptables -t nat -I POSTROUTING -d 10.210.0.12 ! -s 10.210.0.0/24 -j MASQUERADE
-iptables -I FORWARD -d 10.210.0.12 -j ACCEPT
-```
-
-Where `10.210.0.12` is public IP of one of your k8s minions and `32005` is nodePort of `kubernetes-dashboard` service.
-
-* Access:
-
-```bash
-curl 10.210.0.12:9090
-```
-

examples/kubernetes/expose-services/kubedash.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: kubedash-frontend
-  labels:
-    app: kubedash-frontend
-    tier: frontend
-spec:
-  externalIPs:
-  - 10.210.0.12
-  - 10.210.0.13
-  - 10.210.0.14
-  - 10.210.0.15
-  - 10.210.0.16
-  - 10.210.0.17
-  ports:
-  - name: http
-    port: 8289
-    protocol: TCP
-    targetPort: 8289
-  selector:
-    name: kubedash

examples/kubernetes/expose-services/kubernetes-dashboard.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: dashboard-frontend
-  labels:
-    app: dashboard-frontend
-    tier: frontend
-spec:
-  externalIPs:
-  - 10.210.0.12
-  - 10.210.0.13
-  - 10.210.0.14
-  - 10.210.0.15
-  - 10.210.0.16
-  - 10.210.0.17
-  ports:
-  - name: http
-    port: 9090
-    protocol: TCP
-    targetPort: 9090
-  selector:
-    app: kubernetes-dashboard

examples/kubernetes/external-nginx/README.md

@@ -1,18 +0,0 @@
-Nginx example with external IPs
-===============================
-
-* Edit `nginx-frontend.yaml` and update `externalIPs` to the list of external IPs of your k8s minions
-
-* Deploy:
-
-```bash
-kubectl create -f nginx-backends.yaml
-kubectl create -f nginx-frontend.yaml
-```
-
-* Check:
-
-```bash
-curl $ANY_MINION_EXTERNAL_IP
-```
-

examples/kubernetes/external-nginx/nginx-backends.yaml

@@ -1,24 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: nginx-backend
-spec:
-  replicas: 3
-  template:
-    metadata:
-      labels:
-        app: nginx-backend
-        tier: backend
-    spec:
-      containers:
-      - name: nginx
-        image: nginx
-        resources:
-          requests:
-            cpu: 100m
-            memory: 100Mi
-        env:
-        - name: GET_HOSTS_FROM
-          value: dns
-        ports:
-        - containerPort: 80

examples/kubernetes/external-nginx/nginx-frontend.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx-frontend
-  labels:
-    app: nginx-frontend
-    tier: frontend
-spec:
-  externalIPs:
-  - 10.210.0.12
-  - 10.210.0.13
-  - 10.210.0.14
-  - 10.210.0.15
-  - 10.210.0.16
-  - 10.210.0.17
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-    targetPort: 80
-  selector:
-    app: nginx-backend

nodes_to_inv.py

@@ -1,97 +0,0 @@
-#!/usr/bin/env python
-
-# A simple dynamic replacemant of 'kargo prepare'
-# Generates ansible inventory from a list of IPs in 'nodes' file.
-
-import argparse
-import json
-import os
-import yaml
-
-def read_nodes_from_file(filename):
-    f = open(filename, 'r')
-    content = [x.strip('\n') for x in f.readlines()]
-    return content
-
-def read_vars_from_file(src="/root/kargo/inventory/group_vars/all.yml"):
-    with open(src, 'r') as f:
-        content = yaml.load(f)
-    return content
-
-def nodes_to_hash(nodes_list, masters, group_vars):
-    nodes = {
-          'all': {
-              'hosts': [],
-              'vars': group_vars
-          },
-          'etcd': {
-              'hosts': [],
-          },
-          'kube-master': {
-              'hosts': [],
-          },
-          'kube-node': {
-              'hosts': [],
-          },
-          'k8s-cluster': {
-              'children': ['kube-node', 'kube-master']
-          },
-          '_meta': {
-              'hostvars': {}
-          }
-    }
-    i = 1
-
-    for node_ip in nodes_list:
-        node_name = "node%s" % i
-        nodes['all']['hosts'].append(node_name)
-        nodes['_meta']['hostvars'][node_name] = {
-            'ansible_ssh_host': node_ip,
-            'ip': node_ip,
-        }
-        nodes['kube-node']['hosts'].append(node_name)
-        if i <= masters:
-            nodes['kube-master']['hosts'].append(node_name)
-        if i <= 3:
-            nodes['etcd']['hosts'].append(node_name)
-        i += 1
-
-    return nodes
-
-def main():
-    parser = argparse.ArgumentParser(description='Kargo inventory simulator')
-    parser.add_argument('--list', action='store_true')
-    parser.add_argument('--host', default=False)
-    args = parser.parse_args()
-
-    # Read params from ENV since ansible does not support passing args to dynamic inv scripts
-    if os.environ.get('K8S_NODES_FILE'):
-        nodes_file = os.environ['K8S_NODES_FILE']
-    else:
-        nodes_file = 'nodes'
-
-    if os.environ.get('K8S_MASTERS'):
-        masters = int(os.environ['K8S_MASTERS'])
-    else:
-        masters = 2
-
-    if os.environ.get('KARGO_GROUP_VARS'):
-        vars_file = os.environ['KARGO_GROUP_VARS']
-    else:
-        vars_file = "/root/kargo/inventory/group_vars/all.yml"
-
-    nodes_list = read_nodes_from_file(nodes_file)
-
-    if len(nodes_list) < 3:
-        print "Error: requires at least 3 nodes"
-        return
-
-    nodes = nodes_to_hash(nodes_list, masters, read_vars_from_file(vars_file))
-
-    if args.host:
-        print json.dumps(nodes['_meta']['hostvars'][args.host])
-    else:
-        print json.dumps(nodes)
-
-if __name__ == "__main__":
-    main()

playbooks/bootstrap-nodes.yaml

@@ -1,17 +0,0 @@
-- hosts: all
-  tasks:
-    - name: Install packages
-      package: name={{ item }} state=latest
-      with_items:
-        - python-pip
-        - screen
-        - vim
-        - telnet
-        - tcpdump
-        - traceroute
-        - iperf3
-        - nmap
-        - ethtool
-        - curl
-        - git
-        - dnsutils

playbooks/ccp-build.yaml

@@ -1,69 +0,0 @@
-- hosts: kube-master
-
-  pre_tasks:
-
-    - name: Download fuel-ccp
-      git:
-        repo: https://git.openstack.org/openstack/fuel-ccp
-        dest: /usr/local/src/fuel-ccp
-        version: master
-
-    - name: Upload ccp configs to master nodes
-      synchronize:
-        src: ../ccp/
-        dest: /root/ccp/
-
-  tasks:
-
-    - name: Install CCP cli tool
-      shell: pip install -U fuel-ccp/
-      args:
-        chdir: /usr/local/src
-        creates: /usr/local/bin/mcp-microservices
-
-    - name: Get pods
-      shell: kubectl get pods
-      register: get_pod
-      run_once: true
-
-    - name: Get services
-      shell: kubectl get svc
-      register: get_svc
-      run_once: true
-
-    - name: Create registry pod
-      shell: kubectl create -f registry_pod.yaml
-      args:
-        chdir: /root/ccp
-      run_once: true
-      when: get_pod.stdout.find('registry') == -1
-
-    - name: Create registry svc
-      shell: kubectl create -f registry_svc.yaml
-      args:
-        chdir: /root/ccp
-      run_once: true
-      when: get_svc.stdout.find('registry') == -1
-
-    - name: Fetch CCP images
-      shell: mcp-microservices --config-file=/root/ccp/ccp.conf fetch
-      run_once: true
-
-#    - name: Patch fuel-ccp-neutron
-#      run_once: true
-#      args:
-#        chdir: /root/microservices-repos/fuel-ccp-neutron
-#      shell: git fetch https://git.openstack.org/openstack/fuel-ccp-neutron {{ item }} && git cherry-pick FETCH_HEAD
-#      with_items:
-#        - "refs/changes/96/340496/6"
-
-    - name: Build CCP images
-      shell: mcp-microservices --config-file=/root/ccp/ccp.conf build
-      run_once: true
-
-- hosts: k8s-cluster
-
-  tasks:
-
-    - name: Check number of built images
-      shell: test $(curl -s 127.0.0.1:31500/v2/_catalog | python -mjson.tool | grep mcp/ | wc -l) -ge 29

playbooks/ccp-deploy.yaml

@@ -1,27 +0,0 @@
-- hosts: kube-master
-
-  pre_tasks:
-
-    - name: Rsync CCP configs
-      synchronize:
-        src: ../ccp/
-        dest: /root/ccp/
-
-  tasks:
-    - name: Label nodes
-      shell: ./label-nodes.sh
-      args:
-        chdir: /root/ccp
-      run_once: true
-
-    - name: Get namespaces
-      shell: kubectl get namespace
-      register: get_ns
-      run_once: true
-
-    - name: Deploy CCP
-      shell: mcp-microservices --config-file=/root/ccp/ccp.conf deploy
-      args:
-        chdir: /root/ccp
-      run_once: true
-      when: get_ns.stdout.find('openstack') == -1

playbooks/ipro_for_cluster_ips.yaml

@@ -1,24 +0,0 @@
-# FXIME: add persistent routing rule
-- hosts: kube-master
-  tasks:
-    - name: Get kube service net
-      shell: grep KUBE_SERVICE_ADDRESSES /etc/kubernetes/kube-apiserver.env | grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]{1,2}\b"
-      register: kube_service_addresses
-      run_once: true
-- hosts: all
-  tasks:
-    - name: Get local IP
-      shell: "calicoctl status | grep IP: | awk '{print $2}'"
-      register: local_ip
-    - name: Get route
-      shell: ip ro ls | grep "^{{ hostvars[groups['kube-master'][0]]['kube_service_addresses']['stdout'] }}" || echo ""
-      register: local_route
-    - name: Clean up route
-      shell: ip ro del {{ hostvars[groups['kube-master'][0]]['kube_service_addresses']['stdout'] }} || true
-      when: local_route.stdout.find('{{ local_ip.stdout }}') == -1
-    - name: Setup route
-      shell: ip ro add {{ hostvars[groups['kube-master'][0]]['kube_service_addresses']['stdout'] }} via {{ local_ip.stdout }}
-      when: local_route.stdout.find('{{ local_ip.stdout }}') == -1
-    - name: Add openstack namespace to resolv.conf
-      shell: grep openstack.svc.cluster.local /etc/resolv.conf || sed '/^search / s/$/ openstack.svc.cluster.local/' -i /etc/resolv.conf
-

playbooks/kubedash.yaml

@@ -1,5 +0,0 @@
-- hosts: kube-master
-  tasks:
-    - name: setup-kubedns
-      shell: kpm deploy kube-system/kubedash --namespace=kube-system
-      run_once: true

playbooks/kubedashboard.yaml

@@ -1,5 +0,0 @@
-- hosts: kube-master
-  tasks:
-    - name: setup-kubedns
-      shell: kpm deploy kube-system/kubernetes-dashboard --namespace=kube-system
-      run_once: true

playbooks/kubedns.yaml

@@ -1,5 +0,0 @@
-- hosts: kube-master
-  tasks:
-    - name: setup-kubedns
-      shell: kpm deploy kube-system/kubedns --namespace=kube-system
-      run_once: true

requirements.yml

@@ -0,0 +1,41 @@
+---
+- src: https://github.com/ansibl8s/k8s-common.git
+  path: roles/apps
+  version: v1.0
+
+- src: https://github.com/ansibl8s/k8s-kubedns.git
+  path: roles/apps
+  version: v1.0
+
+#- src: https://github.com/ansibl8s/k8s-kube-ui.git
+#  path: roles/apps
+#  version: v1.0
+#
+#- src: https://github.com/ansibl8s/k8s-fabric8.git
+#  path: roles/apps
+#  version: v1.0
+#
+#- src: https://github.com/ansibl8s/k8s-elasticsearch.git
+#  path: roles/apps
+#  # version: v1.0
+#
+#- src: https://github.com/ansibl8s/k8s-redis.git
+#  path: roles/apps
+#  # version: v1.0
+#
+#- src: https://github.com/ansibl8s/k8s-memcached.git
+#  path: roles/apps
+#  version: v1.0
+#
+#- src: https://github.com/ansibl8s/k8s-postgres.git
+#  path: roles/apps
+#  version: v1.0
+#
+#- src: https://github.com/ansibl8s/k8s-heapster.git
+#  path: roles/apps
+#
+#- src: https://github.com/ansibl8s/k8s-influxdb.git
+#  path: roles/apps
+#
+#- src: https://github.com/ansibl8s/k8s-kubedash.git
+#  path: roles/apps

roles/dnsmasq/files/dhclient_nodnsupdate

@@ -0,0 +1,4 @@
+#!/bin/sh
+make_resolv_conf() {
+    :
+}

roles/dnsmasq/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: restart dnsmasq
+  command: systemctl restart dnsmasq

roles/dnsmasq/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- name: populate inventory into hosts file
+  lineinfile:
+    dest: /etc/hosts
+    regexp: "^{{ hostvars[item].ansible_default_ipv4.address }} {{ item }}$"
+    line: "{{ hostvars[item].ansible_default_ipv4.address }} {{ item }}"
+    state: present
+  when: hostvars[item].ansible_default_ipv4.address is defined
+  with_items: groups['all']
+
+- name: clean hosts file
+  lineinfile:
+    dest: /etc/hosts
+    regexp: "{{ item }}"
+    state: absent
+  with_items:
+    - '^127\.0\.0\.1(\s+){{ inventory_hostname }}.*'
+    - '^::1(\s+){{ inventory_hostname }}.*'
+
+- name: install dnsmasq and bindr9utils
+  apt:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - dnsmasq
+    - bind9utils
+  when: inventory_hostname in groups['kube-master'][0]
+
+- name: ensure dnsmasq.d directory exists
+  file:
+    path: /etc/dnsmasq.d
+    state: directory
+  when: inventory_hostname in groups['kube-master'][0]
+
+- name: configure dnsmasq
+  template:
+    src: 01-kube-dns.conf.j2
+    dest: /etc/dnsmasq.d/01-kube-dns.conf
+    mode: 755
+  notify:
+    - restart dnsmasq
+  when: inventory_hostname in groups['kube-master'][0]
+
+- name: enable dnsmasq
+  service:
+    name: dnsmasq
+    state: started
+    enabled: yes
+  when: inventory_hostname in groups['kube-master'][0]
+
+- name: update resolv.conf with new DNS setup
+  template:
+    src: resolv.conf.j2
+    dest: /etc/resolv.conf
+    mode: 644
+
+- name: disable resolv.conf modification by dhclient
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate mode=u+x

roles/dnsmasq/templates/01-kube-dns.conf.j2

@@ -0,0 +1,19 @@
+#Listen on all interfaces
+interface=*
+
+addn-hosts=/etc/hosts
+
+bogus-priv
+
+#Set upstream dns servers
+{% if upstream_dns_servers is defined %}
+{% for srv in upstream_dns_servers %}
+server={{ srv }}
+{% endfor %}
+{% else %}
+ server=8.8.8.8
+ server=8.8.4.4
+{% endif %}
+
+# Forward k8s domain to kube-dns
+server=/{{ dns_domain }}/{{ dns_server }}

roles/dnsmasq/templates/resolv.conf.j2

@@ -0,0 +1,5 @@
+; generated by ansible
+search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | join(' ') }}
+{% for host in groups['kube-master'] %}
+nameserver {{ hostvars[host]['ansible_default_ipv4']['address'] }}
+{% endfor %}

roles/docker/files/systemd-docker.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+After=network.target docker.socket
+Requires=docker.socket
+
+[Service]
+EnvironmentFile=-/etc/default/docker
+Type=notify
+ExecStart=/usr/bin/docker daemon -H fd:// $DOCKER_OPTS
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target

roles/docker/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: restart docker
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart docker service
+
+- name: reload systemd
+  shell: systemctl daemon-reload
+
+- name: restart docker service
+  service: name=docker state=restarted

roles/docker/tasks/configure.yml

@@ -0,0 +1,16 @@
+---
+- name: enable docker
+  service:
+    name: docker
+    enabled: yes
+    state: started
+  tags:
+    - docker
+
+#- name: login to arkena's docker registry
+#  shell : >
+#    docker login --username={{ dockerhub_user }}
+#    --password={{ dockerhub_pass }}
+#    --email={{ dockerhub_email }}
+
+- meta: flush_handlers

roles/docker/tasks/install.yml

@@ -0,0 +1,24 @@
+---
+- name: Install prerequisites for https transport
+  apt: pkg={{ item }} state=present update_cache=yes
+  with_items:
+    - apt-transport-https
+    - ca-certificates
+
+- name: Configure docker apt repository
+  template: src=docker.list.j2 dest=/etc/apt/sources.list.d/docker.list backup=yes
+
+- name: Install docker-engine
+  apt: pkg={{ item }} state=present force=yes update_cache=yes
+  with_items:
+    - aufs-tools
+    - cgroupfs-mount
+    - docker-engine=1.8.3-0~{{ ansible_distribution_release }}
+
+- name: Copy default docker configuration
+  template: src=default-docker.j2 dest=/etc/default/docker backup=yes
+  notify: restart docker
+
+- name: Copy Docker systemd unit file
+  copy: src=systemd-docker.service dest=/lib/systemd/system/docker.service backup=yes
+  notify: restart docker

roles/docker/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- include: install.yml
+- include: configure.yml

roles/docker/templates/default-docker.j2

@@ -0,0 +1,13 @@
+# Docker Upstart and SysVinit configuration file
+
+# Customize location of Docker binary (especially for development testing).
+#DOCKER="/usr/local/bin/docker"
+
+# Use DOCKER_OPTS to modify the daemon startup options.
+#DOCKER_OPTS=""
+
+# If you need Docker to use an HTTP proxy, it can also be specified here.
+#export http_proxy="http://127.0.0.1:3128/"
+
+# This is also a handy place to tweak where Docker's temporary files go.
+#export TMPDIR="/mnt/bigdrive/docker-tmp"

roles/docker/templates/docker.list.j2

@@ -0,0 +1 @@
+deb https://apt.dockerproject.org/repo {{ansible_distribution|lower}}-{{ ansible_distribution_release}} main

roles/docker/vars/main.yml

@@ -0,0 +1,4 @@
+---
+#dockerhub_user:
+#dockerhub_pass:
+#dockerhub_email:

roles/download/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+etcd_download_url: https://github.com/coreos/etcd/releases/download
+flannel_download_url: https://github.com/coreos/flannel/releases/download
+kube_download_url: https://github.com/GoogleCloudPlatform/kubernetes/releases/download
+calico_download_url: https://github.com/Metaswitch/calico-docker/releases/download
+
+etcd_version: v2.2.2
+flannel_version: 0.5.5
+
+kube_version: v1.1.2
+kube_sha1: 69d110d371752c6492d2f8695aa7a47be5b6ed4e
+
+calico_version: v0.11.0

roles/download/tasks/calico.yml

@@ -0,0 +1,21 @@
+---
+- name: Create calico release directory
+  local_action: file
+     path={{ local_release_dir }}/calico/bin
+     recurse=yes
+     state=directory
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Check if calicoctl has been downloaded
+  local_action: stat
+     path={{ local_release_dir }}/calico/bin/calicoctl
+  register: c_tar
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+# issues with get_url module and redirects, to be tested again in the near future
+- name: Download calico
+  local_action: shell
+    curl -o {{ local_release_dir }}/calico/bin/calicoctl -Ls {{ calico_download_url }}/{{ calico_version }}/calicoctl
+  when: not c_tar.stat.exists
+  register: dl_calico
+  delegate_to: "{{ groups['kube-master'][0] }}"

roles/download/tasks/etcd.yml

@@ -0,0 +1,42 @@
+---
+- name: Create etcd release directory
+  local_action: file
+     path={{ local_release_dir }}/etcd/bin
+     recurse=yes
+     state=directory
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Check if etcd release archive has been downloaded
+  local_action: stat
+     path={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz
+  register: e_tar
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+# issues with get_url module and redirects, to be tested again in the near future
+- name: Download etcd
+  local_action: shell
+    curl -o {{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz -Ls {{ etcd_download_url }}/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz
+  when: not e_tar.stat.exists
+  register: dl_etcd
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Extract etcd archive
+  local_action: unarchive
+     src={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz
+     dest={{ local_release_dir }}/etcd copy=no
+  when: dl_etcd|changed
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Pick up only etcd binaries
+  local_action: copy
+     src={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/{{ item }}
+     dest={{ local_release_dir }}/etcd/bin
+  with_items:
+    - etcdctl
+    - etcd
+  when: dl_etcd|changed
+
+- name: Delete unused etcd files
+  local_action: file
+     path={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64 state=absent
+  when: dl_etcd|changed

roles/download/tasks/flannel.yml

@@ -0,0 +1,39 @@
+---
+- name: Create flannel release directory
+  local_action: file
+     path={{ local_release_dir }}/flannel
+     recurse=yes
+     state=directory
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Check if flannel release archive has been downloaded
+  local_action: stat
+     path={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz
+  register: f_tar
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+# issues with get_url module and redirects, to be tested again in the near future
+- name: Download flannel
+  local_action: shell
+    curl -o {{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz -Ls {{ flannel_download_url }}/v{{ flannel_version }}/flannel-{{ flannel_version }}-linux-amd64.tar.gz
+  when: not f_tar.stat.exists
+  register: dl_flannel
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Extract flannel archive
+  local_action: unarchive
+     src={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz
+     dest={{ local_release_dir }}/flannel copy=no
+  when: dl_flannel|changed
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Pick up only flannel binaries
+  local_action: copy
+     src={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}/flanneld
+     dest={{ local_release_dir }}/flannel/bin
+  when: dl_flannel|changed
+
+- name: Delete unused flannel files
+  local_action: file
+     path={{ local_release_dir }}/flannel/flannel-{{ flannel_version }} state=absent
+  when: dl_flannel|changed

roles/download/tasks/kubernetes.yml

@@ -0,0 +1,47 @@
+---
+- name: Create kubernetes release directory
+  local_action: file
+     path={{ local_release_dir }}/kubernetes
+     state=directory
+
+- name: Check if kubernetes release archive has been downloaded
+  local_action: stat
+     path={{ local_release_dir }}/kubernetes/kubernetes.tar.gz
+  register: k_tar
+
+# issues with get_url module and redirects, to be tested again in the near future
+- name: Download kubernetes
+  local_action: shell
+    curl -o {{ local_release_dir }}/kubernetes/kubernetes.tar.gz -Ls {{ kube_download_url }}/{{ kube_version }}/kubernetes.tar.gz
+  when: not k_tar.stat.exists or k_tar.stat.checksum != "{{ kube_sha1 }}"
+  register: dl_kube
+
+- name: Compare kubernetes archive checksum
+  local_action: stat
+     path={{ local_release_dir }}/kubernetes/kubernetes.tar.gz
+  register: k_tar
+  failed_when: k_tar.stat.checksum != "{{ kube_sha1 }}"
+  when: dl_kube|changed
+
+- name: Extract kubernetes archive
+  local_action: unarchive
+     src={{ local_release_dir }}/kubernetes/kubernetes.tar.gz
+     dest={{ local_release_dir }}/kubernetes copy=no
+  when: dl_kube|changed
+
+- name: Extract kubernetes binaries archive
+  local_action: unarchive
+     src={{ local_release_dir }}/kubernetes/kubernetes/server/kubernetes-server-linux-amd64.tar.gz
+     dest={{ local_release_dir }}/kubernetes copy=no
+  when: dl_kube|changed
+
+- name: Pick up only kubernetes binaries
+  local_action: synchronize
+     src={{ local_release_dir }}/kubernetes/kubernetes/server/bin
+     dest={{ local_release_dir }}/kubernetes
+  when: dl_kube|changed
+
+- name: Delete unused kubernetes files
+  local_action: file
+     path={{ local_release_dir }}/kubernetes/kubernetes state=absent
+  when: dl_kube|changed

roles/download/tasks/main.yml

@@ -0,0 +1,5 @@
+---
+- include: kubernetes.yml
+- include: etcd.yml
+- include: calico.yml
+- include: flannel.yml

roles/etcd/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+- name: restart daemons
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart etcd2
+
+- name: reload systemd
+  command: systemctl daemon-reload
+
+- name: restart etcd2
+  service: name=etcd2 state=restarted
+
+- name: Save iptables rules
+  command: service iptables save

roles/etcd/tasks/configure.yml

@@ -0,0 +1,15 @@
+---
+- name: Disable ferm
+  service: name=ferm state=stopped enabled=no
+
+- name: Create etcd2 environment vars dir
+  file: path=/etc/systemd/system/etcd2.service.d state=directory
+
+- name: Write etcd2 config file
+  template: src=etcd2.j2 dest=/etc/systemd/system/etcd2.service.d/10-etcd2-cluster.conf backup=yes
+  notify:
+    - reload systemd
+    - restart etcd2
+
+- name: Ensure etcd2 is running
+  service: name=etcd2 state=started enabled=yes

roles/etcd/tasks/install.yml

@@ -0,0 +1,25 @@
+---
+- name: Create etcd user
+  user: name=etcd shell=/bin/nologin home=/var/lib/etcd2
+
+- name: Install etcd binaries
+  copy:
+     src={{ local_release_dir }}/etcd/bin/{{ item }}
+     dest={{ bin_dir }}
+     owner=etcd
+     mode=u+x
+  with_items:
+    - etcdctl
+    - etcd
+  notify:
+    - restart daemons
+
+- name: Create etcd2 binary symlink
+  file: src=/usr/local/bin/etcd dest=/usr/local/bin/etcd2 state=link
+
+- name: Copy etcd2.service systemd file
+  template:
+    src: systemd-etcd2.service.j2
+    dest: /lib/systemd/system/etcd2.service
+    backup: yes
+  notify: restart daemons

roles/etcd/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- include: install.yml
+- include: configure.yml

roles/etcd/templates/etcd2.j2

@@ -0,0 +1,17 @@
+# etcd2.0
+[Service]
+{% if inventory_hostname in groups['kube-master'] %}
+Environment="ETCD_ADVERTISE_CLIENT_URLS=http://{{ ansible_default_ipv4.address }}:2379,http://{{ ansible_default_ipv4.address }}:4001"
+Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{ ansible_default_ipv4.address }}:2380"
+Environment="ETCD_INITIAL_CLUSTER=master=http://{{ ansible_default_ipv4.address }}:2380"
+Environment="ETCD_INITIAL_CLUSTER_STATE=new"
+Environment="ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd"
+Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001"
+Environment="ETCD_LISTEN_PEER_URLS=http://:2380,http://{{ ansible_default_ipv4.address }}:7001"
+Environment="ETCD_NAME=master"
+{% else %}
+Environment="ETCD_ADVERTISE_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001"
+Environment="ETCD_INITIAL_CLUSTER=master=http://{{ groups['kube-master'][0] }}:2380"
+Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001"
+Environment="ETCD_PROXY=on"
+{% endif %}

roles/etcd/templates/systemd-etcd2.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=etcd2
+Conflicts=etcd.service
+
+[Service]
+User=etcd
+Environment=ETCD_DATA_DIR=/var/lib/etcd2
+Environment=ETCD_NAME=%m
+ExecStart={{ bin_dir }}/etcd2
+Restart=always
+RestartSec=10s
+LimitNOFILE=40000
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/common/defaults/main.yml

@@ -0,0 +1,44 @@
+# This directory is where all the additional scripts go
+# that Kubernetes normally puts in /srv/kubernetes.
+# This puts them in a sane location
+kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
+
+# This directory is where all the additional config stuff goes
+# the kubernetes normally puts in /srv/kubernets.
+# This puts them in a sane location.
+# Editting this value will almost surely break something. Don't
+# change it. Things like the systemd scripts are hard coded to
+# look in here. Don't do it.
+kube_config_dir: /etc/kubernetes
+
+# This is where all the cert scripts and certs will be located
+kube_cert_dir: "{{ kube_config_dir }}/certs"
+
+# This is where all of the bearer tokens will be stored
+kube_token_dir: "{{ kube_config_dir }}/tokens"
+
+# This is where to save basic auth file
+kube_users_dir: "{{ kube_config_dir }}/users"
+
+# This is where you can drop yaml/json files and the kubelet will run those
+# pods on startup
+kube_manifest_dir: "{{ kube_config_dir }}/manifests"
+
+# This is the group that the cert creation scripts chgrp the
+# cert files to. Not really changable...
+kube_cert_group: kube-cert
+
+dns_domain: "{{ cluster_name }}"
+
+kube_proxy_mode: iptables
+
+# IP address of the DNS server.
+# Kubernetes will create a pod with several containers, serving as the DNS
+# server and expose it under this IP address. The IP address must be from
+# the range specified as kube_service_addresses. This magic will actually
+# pick the 10th ip address in the kube_service_addresses range and use that.
+# dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
+
+# kube_api_runtime_config:
+#   - extensions/v1beta1/daemonsets=true
+#   - extensions/v1beta1/deployments=true

roles/kubernetes/common/files/kube-gen-token.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+token_dir=${TOKEN_DIR:-/var/srv/kubernetes}
+token_file="${token_dir}/known_tokens.csv"
+
+create_accounts=($@)
+
+touch "${token_file}"
+for account in "${create_accounts[@]}"; do
+  if grep ",${account}," "${token_file}" ; then
+    continue
+  fi
+  token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+  echo "${token},${account},${account}" >> "${token_file}"
+  echo "${token}" > "${token_dir}/${account}.token"
+  echo "Added ${account}"
+done

roles/kubernetes/common/files/make-ca-cert.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Caller should set in the ev:
+# MASTER_IP - this may be an ip or things like "_use_gce_external_ip_"
+# DNS_DOMAIN - which will be passed to minions in --cluster_domain
+# SERVICE_CLUSTER_IP_RANGE - where all service IPs are allocated
+# MASTER_NAME - I'm not sure what it is...
+
+# Also the following will be respected
+# CERT_DIR - where to place the finished certs
+# CERT_GROUP - who the group owner of the cert files should be
+
+cert_ip="${MASTER_IP:="${1}"}"
+master_name="${MASTER_NAME:="kubernetes"}"
+service_range="${SERVICE_CLUSTER_IP_RANGE:="10.0.0.0/16"}"
+dns_domain="${DNS_DOMAIN:="cluster.local"}"
+cert_dir="${CERT_DIR:-"/srv/kubernetes"}"
+cert_group="${CERT_GROUP:="kube-cert"}"
+
+# The following certificate pairs are created:
+#
+#  - ca (the cluster's certificate authority)
+#  - server
+#  - kubelet
+#  - kubecfg (for kubectl)
+#
+# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate
+# the certs that we need.
+
+# TODO: Add support for discovery on other providers?
+if [ "$cert_ip" == "_use_gce_external_ip_" ]; then
+  cert_ip=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
+fi
+
+if [ "$cert_ip" == "_use_aws_external_ip_" ]; then
+  cert_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
+fi
+
+if [ "$cert_ip" == "_use_azure_dns_name_" ]; then
+  cert_ip=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
+fi
+
+tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+# TODO: For now, this is a patched tool that makes subject-alt-name work, when
+# the fix is upstream  move back to the upstream easyrsa.  This is cached in GCS
+# but is originally taken from:
+#   https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
+#
+# To update, do the following:
+# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
+# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
+# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
+#
+# Due to GCS caching of public objects, it may take time for this to be widely
+# distributed.
+
+# Calculate the first ip address in the service range
+octects=($(echo "${service_range}" | sed -e 's|/.*||' -e 's/\./ /g'))
+((octects[3]+=1))
+service_ip=$(echo "${octects[*]}" | sed 's/ /./g')
+
+# Determine appropriete subject alt names
+sans="IP:${cert_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${dns_domain},DNS:${master_name}"
+
+curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
+tar xzf easy-rsa.tar.gz > /dev/null
+cd easy-rsa-master/easyrsa3
+
+(./easyrsa init-pki > /dev/null 2>&1
+ ./easyrsa --batch "--req-cn=${cert_ip}@$(date +%s)" build-ca nopass > /dev/null 2>&1
+ ./easyrsa --subject-alt-name="${sans}" build-server-full "${master_name}" nopass > /dev/null 2>&1
+ ./easyrsa build-client-full kubelet nopass > /dev/null 2>&1
+ ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1) || {
+ # If there was an error in the subshell, just die.
+ # TODO(roberthbailey): add better error handling here
+ echo "=== Failed to generate certificates: Aborting ==="
+ exit 2
+ }
+
+mkdir -p "$cert_dir"
+
+cp -p pki/ca.crt "${cert_dir}/ca.crt"
+cp -p "pki/issued/${master_name}.crt" "${cert_dir}/server.crt" > /dev/null 2>&1
+cp -p "pki/private/${master_name}.key" "${cert_dir}/server.key" > /dev/null 2>&1
+cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
+cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"
+cp -p pki/issued/kubelet.crt "${cert_dir}/kubelet.crt"
+cp -p pki/private/kubelet.key "${cert_dir}/kubelet.key"
+
+CERTS=("ca.crt" "server.key" "server.crt" "kubelet.key" "kubelet.crt" "kubecfg.key" "kubecfg.crt")
+for cert in "${CERTS[@]}"; do
+  chgrp "${cert_group}" "${cert_dir}/${cert}"
+  chmod 660 "${cert_dir}/${cert}"
+done

roles/kubernetes/common/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: etcd }

roles/kubernetes/common/tasks/gen_certs.yml

@@ -0,0 +1,42 @@
+---
+#- name: Get create ca cert script from Kubernetes
+#  get_url:
+#    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
+#    dest={{ kube_script_dir }}/make-ca-cert.sh mode=0500
+#    force=yes
+
+- name: certs | install cert generation script
+  copy:
+    src=make-ca-cert.sh
+    dest={{ kube_script_dir }}
+    mode=0500
+  changed_when: false
+
+# FIXME This only generates a cert for one master...
+- name: certs | run cert generation script
+  command:
+    "{{ kube_script_dir }}/make-ca-cert.sh {{ inventory_hostname }}"
+  args:
+    creates: "{{ kube_cert_dir }}/server.crt"
+  environment:
+    MASTER_IP: "{{ hostvars[inventory_hostname]['ip'] | default(hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}"
+    MASTER_NAME: "{{ inventory_hostname }}"
+    DNS_DOMAIN: "{{ dns_domain }}"
+    SERVICE_CLUSTER_IP_RANGE: "{{ kube_service_addresses }}"
+    CERT_DIR: "{{ kube_cert_dir }}"
+    CERT_GROUP: "{{ kube_cert_group }}"
+
+- name: certs | check certificate permissions
+  file:
+    path={{ item }}
+    group={{ kube_cert_group }}
+    owner=kube
+    mode=0440
+  with_items:
+    - "{{ kube_cert_dir }}/ca.crt"
+    - "{{ kube_cert_dir }}/server.crt"
+    - "{{ kube_cert_dir }}/server.key"
+    - "{{ kube_cert_dir }}/kubecfg.crt"
+    - "{{ kube_cert_dir }}/kubecfg.key"
+    - "{{ kube_cert_dir }}/kubelet.crt"
+    - "{{ kube_cert_dir }}/kubelet.key"

roles/kubernetes/common/tasks/gen_tokens.yml

@@ -0,0 +1,30 @@
+---
+- name: tokens | copy the token gen script
+  copy:
+    src=kube-gen-token.sh
+    dest={{ kube_script_dir }}
+    mode=u+x
+
+- name: tokens | generate tokens for master components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ "system:controller_manager", "system:scheduler", "system:kubectl", 'system:proxy' ]
+    - "{{ groups['kube-master'][0] }}"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  notify:
+    - restart daemons
+
+- name: tokens | generate tokens for node components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ 'system:kubelet', 'system:proxy' ]
+    - "{{ groups['kube-node'] }}"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  notify:
+    - restart daemons

roles/kubernetes/common/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+- name: define alias command for kubectl all
+  lineinfile:
+    dest=/etc/bash.bashrc
+    line="alias kball='{{ bin_dir }}/kubectl --all-namespaces -o wide'"
+    regexp='^alias kball=.*$'
+    state=present
+    insertafter=EOF
+    create=True
+
+- name: create kubernetes config directory
+  file: path={{ kube_config_dir }} state=directory
+
+- name: create kubernetes script directory
+  file: path={{ kube_script_dir }} state=directory
+
+- name: Make sure manifest directory exists
+  file: path={{ kube_manifest_dir }} state=directory
+
+- name: write the global config file
+  template:
+    src: config.j2
+    dest: "{{ kube_config_dir }}/config"
+  notify:
+    - restart daemons
+
+- include: secrets.yml
+  tags:
+    - secrets

roles/kubernetes/common/tasks/secrets.yml

@@ -0,0 +1,54 @@
+---
+- name: certs | create system kube-cert groups
+  group: name={{ kube_cert_group }} state=present system=yes
+
+- name: create system kube user
+  user:
+    name=kube
+    comment="Kubernetes user"
+    shell=/sbin/nologin
+    state=present
+    system=yes
+    groups={{ kube_cert_group }}
+
+- name: certs | make sure the certificate directory exits
+  file:
+    path={{ kube_cert_dir }}
+    state=directory
+    mode=o-rwx
+    group={{ kube_cert_group }}
+
+- name: tokens | make sure the tokens directory exits
+  file:
+    path={{ kube_token_dir }}
+    state=directory
+    mode=o-rwx
+    group={{ kube_cert_group }}
+
+- include: gen_certs.yml
+  run_once: true
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Read back the CA certificate
+  slurp:
+    src: "{{ kube_cert_dir }}/ca.crt"
+  register: ca_cert
+  run_once: true
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: certs | register the CA certificate as a fact for later use
+  set_fact:
+    kube_ca_cert: "{{ ca_cert.content|b64decode }}"
+
+- name: certs | write CA certificate everywhere
+  copy: content="{{ kube_ca_cert }}" dest="{{ kube_cert_dir }}/ca.crt"
+  notify:
+    - restart daemons
+
+- debug: msg="{{groups['kube-master'][0]}} == {{inventory_hostname}}"
+  tags:
+    - debug
+
+- include: gen_tokens.yml
+  run_once: true
+  when: inventory_hostname == groups['kube-master'][0]

roles/kubernetes/common/templates/config.j2

@@ -0,0 +1,26 @@
+###
+# kubernetes system config
+#
+# The following values are used to configure various aspects of all
+# kubernetes services, including
+#
+#   kube-apiserver.service
+#   kube-controller-manager.service
+#   kube-scheduler.service
+#   kubelet.service
+#   kube-proxy.service
+
+# Comma separated list of nodes in the etcd cluster
+# KUBE_ETCD_SERVERS="--etcd_servers="
+
+# logging to stderr means we get it in the systemd journal
+KUBE_LOGTOSTDERR="--logtostderr=true"
+
+# journal message level, 0 is debug
+KUBE_LOG_LEVEL="--v=5"
+
+# Should this cluster be allowed to run privileged docker containers
+KUBE_ALLOW_PRIV="--allow_privileged=true"
+
+# How the replication controller, scheduler, and proxy
+KUBE_MASTER="--master=https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}"

roles/kubernetes/master/handlers/main.yml

@@ -0,0 +1,56 @@
+---
+- name: restart daemons
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart reloaded-scheduler
+    - restart reloaded-controller-manager
+    - restart reloaded-apiserver
+    - restart reloaded-proxy
+
+- name: reload systemd
+  command: systemctl daemon-reload
+
+- name: restart apiserver
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart reloaded-apiserver
+
+- name: restart reloaded-apiserver
+  service:
+    name: kube-apiserver
+    state: restarted
+
+- name: restart controller-manager
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart reloaded-controller-manager
+
+- name: restart reloaded-controller-manager
+  service:
+    name: kube-controller-manager
+    state: restarted
+
+- name: restart scheduler
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart reloaded-scheduler
+
+- name: restart reloaded-scheduler
+  service:
+    name: kube-scheduler
+    state: restarted
+
+- name: restart proxy
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart reloaded-proxy
+
+- name: restart reloaded-proxy
+  service:
+    name: kube-proxy
+    state: restarted

roles/kubernetes/master/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: kubernetes/common }

roles/kubernetes/master/tasks/config.yml

@@ -0,0 +1,94 @@
+---
+- name: get the node token values from token files
+  slurp:
+    src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
+  with_items:
+    - "system:controller_manager"
+    - "system:scheduler"
+    - "system:kubectl"
+    - "system:proxy"
+  register: tokens
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Set token facts
+  set_fact:
+    controller_manager_token: "{{ tokens.results[0].content|b64decode }}"
+    scheduler_token: "{{ tokens.results[1].content|b64decode }}"
+    kubectl_token: "{{ tokens.results[2].content|b64decode }}"
+    proxy_token: "{{ tokens.results[3].content|b64decode }}"
+
+- name: write the config files for api server
+  template: src=apiserver.j2 dest={{ kube_config_dir }}/apiserver backup=yes
+  notify:
+    - restart apiserver
+
+- name: write config file for controller-manager
+  template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager backup=yes
+  notify:
+    - restart controller-manager
+
+- name: write the kubecfg (auth) file for controller-manager
+  template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig backup=yes
+  notify:
+    - restart controller-manager
+
+- name: write the config file for scheduler
+  template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler backup=yes
+  notify:
+    - restart scheduler
+
+- name: write the kubecfg (auth) file for scheduler
+  template: src=scheduler.kubeconfig.j2 dest={{ kube_config_dir }}/scheduler.kubeconfig backup=yes
+  notify:
+    - restart scheduler
+
+- name: write the kubecfg (auth) file for kubectl
+  template: src=kubectl.kubeconfig.j2 dest={{ kube_config_dir }}/kubectl.kubeconfig backup=yes
+
+- name: Copy kubectl bash completion
+  copy: src=kubectl_bash_completion.sh dest=/etc/bash_completion.d/kubectl.sh
+
+- name: Create proxy environment vars dir
+  file: path=/etc/systemd/system/kube-proxy.service.d state=directory
+
+- name: Write proxy config file
+  template: src=proxy.j2 dest=/etc/systemd/system/kube-proxy.service.d/10-proxy-cluster.conf backup=yes
+  notify:
+    - restart proxy
+
+- name: write the kubecfg (auth) file for proxy
+  template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig backup=yes
+
+- name: populate users for basic auth in API
+  lineinfile:
+    dest: "{{ kube_users_dir }}/known_users.csv"
+    create: yes
+    line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
+    backup: yes
+  with_dict: "{{ kube_users }}"
+  notify:
+    - restart apiserver
+
+- name: Enable controller-manager
+  service:
+    name: kube-controller-manager
+    enabled: yes
+    state: started
+
+- name: Enable scheduler
+  service:
+    name: kube-scheduler
+    enabled: yes
+    state: started
+
+- name: Enable kube-proxy
+  service:
+    name: kube-proxy
+    enabled: yes
+    state: started
+
+- name: Enable apiserver
+  service:
+    name: kube-apiserver
+    enabled: yes
+    state: started

roles/kubernetes/master/tasks/install.yml

@@ -0,0 +1,34 @@
+---
+- name: Write kube-apiserver systemd init file
+  template: src=systemd-init/kube-apiserver.service.j2 dest=/etc/systemd/system/kube-apiserver.service backup=yes
+  notify: restart apiserver
+
+- name: Write kube-controller-manager systemd init file
+  template: src=systemd-init/kube-controller-manager.service.j2 dest=/etc/systemd/system/kube-controller-manager.service backup=yes
+  notify: restart controller-manager
+
+- name: Write kube-scheduler systemd init file
+  template: src=systemd-init/kube-scheduler.service.j2 dest=/etc/systemd/system/kube-scheduler.service backup=yes
+  notify: restart scheduler
+
+- name: Write kube-proxy systemd init file
+  template: src=systemd-init/kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service backup=yes
+  notify: restart proxy
+
+- name: Install kubernetes binaries
+  copy:
+     src={{ local_release_dir }}/kubernetes/bin/{{ item }}
+     dest={{ bin_dir }}
+     owner=kube
+     mode=u+x
+  with_items:
+    - kube-apiserver
+    - kube-controller-manager
+    - kube-scheduler
+    - kube-proxy
+    - kubectl
+  notify:
+    - restart daemons
+
+- name: Allow apiserver to bind on both secure and insecure ports
+  shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver

roles/kubernetes/master/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- include: install.yml
+- include: config.yml

roles/kubernetes/master/templates/apiserver.j2

@@ -0,0 +1,28 @@
+###
+# kubernetes system config
+#
+# The following values are used to configure the kube-apiserver
+#
+
+# The address on the local server to listen to.
+KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
+
+# The port on the local server to listen on.
+KUBE_API_PORT="--insecure-port={{kube_master_insecure_port}} --secure-port={{ kube_master_port }}"
+
+# KUBELET_PORT="--kubelet_port=10250"
+
+# Address range to use for services
+KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
+
+# Location of the etcd cluster
+KUBE_ETCD_SERVERS="--etcd_servers={% for node in groups['etcd'] %}http://{{ node }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
+
+# default admission control policies
+KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
+
+# RUNTIME API CONFIGURATION (e.g. enable extensions)
+KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}"
+
+# Add you own!
+KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.crt --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/server.crt"

roles/kubernetes/master/templates/controller-manager.j2

@@ -0,0 +1,6 @@
+###
+# The following values are used to configure the kubernetes controller-manager
+
+# defaults from config and apiserver should be adequate
+
+KUBE_CONTROLLER_MANAGER_ARGS="--kubeconfig={{ kube_config_dir }}/controller-manager.kubeconfig --service_account_private_key_file={{ kube_cert_dir }}/server.key --root_ca_file={{ kube_cert_dir }}/ca.crt"

roles/kubernetes/master/templates/controller-manager.kubeconfig.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+current-context: controller-manager-to-{{ cluster_name }}
+preferences: {}
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: controller-manager
+  name: controller-manager-to-{{ cluster_name }}
+users:
+- name: controller-manager
+  user:
+    token: {{ controller_manager_token }}

roles/kubernetes/master/templates/kubectl.kubeconfig.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+current-context: kubectl-to-{{ cluster_name }}
+preferences: {}
+clusters:
+- cluster:
+    certificate-authority-data: {{ kube_ca_cert|b64encode }}
+    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: kubectl
+  name: kubectl-to-{{ cluster_name }}
+users:
+- name: kubectl
+  user:
+    token: {{ kubectl_token }}

roles/kubernetes/master/templates/proxy.j2

@@ -0,0 +1,8 @@
+###
+# kubernetes proxy config
+
+# default config should be adequate
+
+# Add your own!
+[Service]
+Environment="KUBE_PROXY_ARGS=--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig --proxy-mode={{kube_proxy_mode}}"

roles/kubernetes/master/templates/proxy.kubeconfig.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+current-context: proxy-to-{{ cluster_name }}
+preferences: {}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: proxy
+  name: proxy-to-{{ cluster_name }}
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: http://{{ groups['kube-master'][0] }}:{{kube_master_insecure_port}}
+  name: {{ cluster_name }}
+users:
+- name: proxy
+  user:
+    token: {{ proxy_token }}

roles/kubernetes/master/templates/scheduler.j2

@@ -0,0 +1,7 @@
+###
+# kubernetes scheduler config
+
+# default config should be adequate
+
+# Add your own!
+KUBE_SCHEDULER_ARGS="--kubeconfig={{ kube_config_dir }}/scheduler.kubeconfig"

roles/kubernetes/master/templates/scheduler.kubeconfig.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+current-context: scheduler-to-{{ cluster_name }}
+preferences: {}
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: scheduler
+  name: scheduler-to-{{ cluster_name }}
+users:
+- name: scheduler
+  user:
+    token: {{ scheduler_token }}

roles/kubernetes/master/templates/systemd-init/kube-apiserver.service.j2

@@ -0,0 +1,29 @@
+[Unit]
+Description=Kubernetes API Server
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+Requires=etcd2.service
+After=etcd2.service
+
+[Service]
+EnvironmentFile=/etc/network-environment
+EnvironmentFile=-/etc/kubernetes/config
+EnvironmentFile=-/etc/kubernetes/apiserver
+User=kube
+ExecStart={{ bin_dir }}/kube-apiserver \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_ETCD_SERVERS \
+	    $KUBE_API_ADDRESS \
+	    $KUBE_API_PORT \
+	    $KUBELET_PORT \
+	    $KUBE_ALLOW_PRIV \
+	    $KUBE_SERVICE_ADDRESSES \
+	    $KUBE_ADMISSION_CONTROL \
+	    $KUBE_RUNTIME_CONFIG \
+	    $KUBE_API_ARGS
+Restart=on-failure
+Type=notify
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-controller-manager.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description=Kubernetes Controller Manager
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+Requires=etcd2.service
+After=etcd2.service
+
+[Service]
+EnvironmentFile=-/etc/kubernetes/config
+EnvironmentFile=-/etc/kubernetes/controller-manager
+User=kube
+ExecStart={{ bin_dir }}/kube-controller-manager \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_MASTER \
+	    $KUBE_CONTROLLER_MANAGER_ARGS
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-proxy.service.j2

@@ -0,0 +1,22 @@
+[Unit]
+Description=Kubernetes Kube-Proxy Server
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
+After=docker.service calico-node.service
+{% else %}
+After=docker.service
+{% endif %}
+
+[Service]
+EnvironmentFile=/etc/kubernetes/config
+EnvironmentFile=/etc/network-environment
+ExecStart={{ bin_dir }}/kube-proxy \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_MASTER \
+	    $KUBE_PROXY_ARGS
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-scheduler.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description=Kubernetes Scheduler Plugin
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+Requires=etcd2.service
+After=etcd2.service
+
+[Service]
+EnvironmentFile=-/etc/kubernetes/config
+EnvironmentFile=-/etc/kubernetes/scheduler
+User=kube
+ExecStart={{ bin_dir }}/kube-scheduler \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_MASTER \
+	    $KUBE_SCHEDULER_ARGS
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target