diff --git a/.gitignore b/.gitignore index 66c9b48678b..3f7924496fb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .vagrant *.retry -inventory/vagrant_ansible_inventory +**/vagrant_ansible_inventory +inventory/credentials/ inventory/group_vars/fake_hosts.yml inventory/host_vars/ temp @@ -11,9 +12,9 @@ temp *.tfstate *.tfstate.backup contrib/terraform/aws/credentials.tfvars -**/*.sw[pon] /ssh-bastion.conf **/*.sw[pon] +*~ vagrant/ # Byte-compiled / optimized / DLL files @@ -23,7 +24,7 @@ __pycache__/ # Distribution / packaging .Python -artifacts/ +inventory/*/artifacts/ env/ build/ credentials/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c21bb0c434b..d70a479f8dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ variables: GCE_PREEMPTIBLE: "false" ANSIBLE_KEEP_REMOTE_FILES: "1" ANSIBLE_CONFIG: ./tests/ansible.cfg + ANSIBLE_INVENTORY: ./inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini IDEMPOT_CHECK: "false" RESET_CHECK: "false" UPGRADE_TEST: "false" @@ -90,9 +91,9 @@ before_script: - cd tests && make create-${CI_PLATFORM} -s ; cd - # Check out latest tag if testing upgrade - # Uncomment when gitlab kargo repo has tags + # Uncomment when gitlab kubespray repo has tags #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) - - test "${UPGRADE_TEST}" != "false" && git checkout ba0a03a8ba2d97a73d06242ec4bb3c7e2012e58c + - test "${UPGRADE_TEST}" != "false" && git checkout 8b3ce6e418ccf48171eb5b3888ee1af84f8d71ba # Checkout the CI vars file so it is available - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021 @@ -102,14 +103,13 @@ before_script: # Create cluster - > ansible-playbook - -i inventory/sample/hosts.ini + -i ${ANSIBLE_INVENTORY} -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER ${SSH_ARGS} ${LOG_LEVEL} -e @${CI_TEST_VARS} - -e ansible_python_interpreter=${PYPATH} -e ansible_ssh_user=${SSH_USER} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" @@ -122,14 +122,13 @@ before_script: test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml"; git checkout "${CI_BUILD_REF}"; ansible-playbook - -i inventory/sample/hosts.ini + -i ${ANSIBLE_INVENTORY} -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER ${SSH_ARGS} ${LOG_LEVEL} -e @${CI_TEST_VARS} - -e ansible_python_interpreter=${PYPATH} -e ansible_ssh_user=${SSH_USER} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" @@ -139,20 +138,20 @@ before_script: # Tests Cases ## Test Master API - > - ansible-playbook -i inventory/sample/hosts.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL + ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" ## Ping the between 2 pod - - ansible-playbook -i inventory/sample/hosts.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL + - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL ## Advanced DNS checks - - ansible-playbook -i inventory/sample/hosts.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL + - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL ## Idempotency checks 1/5 (repeat deployment) - > if [ "${IDEMPOT_CHECK}" = "true" ]; then ansible-playbook - -i inventory/sample/hosts.ini + -i ${ANSIBLE_INVENTORY} -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER @@ -169,7 +168,7 @@ before_script: - > if [ "${IDEMPOT_CHECK}" = "true" ]; then ansible-playbook - -i inventory/sample/hosts.ini + -i ${ANSIBLE_INVENTORY} -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER @@ -184,7 +183,7 @@ before_script: - > if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then ansible-playbook - -i inventory/sample/hosts.ini + -i ${ANSIBLE_INVENTORY} -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER @@ -201,7 +200,7 @@ before_script: - > if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then ansible-playbook - -i inventory/sample/hosts.ini + -i ${ANSIBLE_INVENTORY} -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER @@ -217,7 +216,7 @@ before_script: ## Idempotency checks 5/5 (Advanced DNS checks) - > if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then - ansible-playbook -i inventory/sample/hosts.ini -e ansible_python_interpreter=${PYPATH} + ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL; @@ -241,6 +240,10 @@ before_script: # stage: deploy-part1 MOVED_TO_GROUP_VARS: "true" +.ubuntu18_flannel_aio_variables: &ubuntu18_flannel_aio_variables +# stage: deploy-part1 + MOVED_TO_GROUP_VARS: "true" + .ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables # stage: deploy-part1 UPGRADE_TEST: "graceful" @@ -257,6 +260,10 @@ before_script: # stage: deploy-special MOVED_TO_GROUP_VARS: "true" +.coreos_cilium_variables: &coreos_cilium_variables +# stage: deploy-special + MOVED_TO_GROUP_VARS: "true" + .ubuntu_cilium_sep_variables: &ubuntu_cilium_sep_variables # stage: deploy-special MOVED_TO_GROUP_VARS: "true" @@ -301,10 +308,18 @@ before_script: # stage: deploy-part1 MOVED_TO_GROUP_VARS: "true" +.coreos_vault_upgrade_variables: &coreos_vault_upgrade_variables +# stage: deploy-part1 + UPGRADE_TEST: "basic" + .ubuntu_flannel_variables: &ubuntu_flannel_variables # stage: deploy-special MOVED_TO_GROUP_VARS: "true" +.opensuse_canal_variables: &opensuse_canal_variables +# stage: deploy-part2 + MOVED_TO_GROUP_VARS: "true" + # Builds for PRs only (premoderated by unit-tests step) and triggers (auto) ### PR JOBS PART1 @@ -320,6 +335,18 @@ gce_coreos-calico-aio: only: [/^pr-.*$/] ### PR JOBS PART2 + +gce_ubuntu18-flannel-aio: + stage: deploy-part2 + <<: *job + <<: *gce + variables: + <<: *ubuntu18_flannel_aio_variables + <<: *gce_variables + when: manual + except: ['triggers'] + only: [/^pr-.*$/] + gce_centos7-flannel-addons: stage: deploy-part2 <<: *job @@ -331,6 +358,17 @@ gce_centos7-flannel-addons: except: ['triggers'] only: [/^pr-.*$/] +gce_centos-weave-kubeadm: + stage: deploy-part2 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *centos_weave_kubeadm_variables + when: on_success + except: ['triggers'] + only: [/^pr-.*$/] + gce_ubuntu-weave-sep: stage: deploy-part2 <<: *job @@ -427,17 +465,6 @@ gce_ubuntu-canal-kubeadm-triggers: when: on_success only: ['triggers'] -gce_centos-weave-kubeadm: - stage: deploy-part2 - <<: *job - <<: *gce - variables: - <<: *gce_variables - <<: *centos_weave_kubeadm_variables - when: manual - except: ['triggers'] - only: ['master', /^pr-.*$/] - gce_centos-weave-kubeadm-triggers: stage: deploy-part2 <<: *job @@ -459,6 +486,17 @@ gce_ubuntu-contiv-sep: except: ['triggers'] only: ['master', /^pr-.*$/] +gce_coreos-cilium: + stage: deploy-special + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *coreos_cilium_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + gce_ubuntu-cilium-sep: stage: deploy-special <<: *job @@ -542,7 +580,7 @@ gce_rhel7-canal-sep: <<: *rhel7_canal_sep_variables when: manual except: ['triggers'] - only: ['master', /^pr-.*$/,] + only: ['master', /^pr-.*$/] gce_rhel7-canal-sep-triggers: stage: deploy-part2 @@ -575,6 +613,17 @@ gce_centos7-calico-ha-triggers: when: on_success only: ['triggers'] +gce_opensuse-canal: + stage: deploy-part2 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *opensuse_canal_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + # no triggers yet https://github.com/kubernetes-incubator/kargo/issues/613 gce_coreos-alpha-weave-ha: stage: deploy-special @@ -609,6 +658,17 @@ gce_ubuntu-vault-sep: except: ['triggers'] only: ['master', /^pr-.*$/] +gce_coreos-vault-upgrade: + stage: deploy-part2 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *coreos_vault_upgrade_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + gce_ubuntu-flannel-sep: stage: deploy-special <<: *job diff --git a/OWNERS b/OWNERS index 6ecbee5c928..1883a49304e 100644 --- a/OWNERS +++ b/OWNERS @@ -1,9 +1,7 @@ # See the OWNERS file documentation: -# https://github.com/kubernetes/kubernetes/blob/master/docs/devel/owners.md +# https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md -owners: - - Smana - - ant31 - - bogdando - - mattymo - - rsmitty +approvers: + - kubespray-approvers +reviewers: + - kubespray-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES new file mode 100644 index 00000000000..a3aa995a7c8 --- /dev/null +++ b/OWNERS_ALIASES @@ -0,0 +1,18 @@ +aliases: + kubespray-approvers: + - ant31 + - mattymo + - atoms + - chadswen + - rsmitty + - bogdando + - bradbeam + - woopstar + - riverzhang + - holser + - smana + kubespray-reviewers: + - jjungnickel + - archifleks + - chapsuk + - mirwan diff --git a/README.md b/README.md index c9ff3f2c9e4..59686019fde 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ -![Kubernetes Logo](https://s28.postimg.org/lf3q4ocpp/k8s.png) +![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-incubator/kubespray/master/docs/img/kubernetes-logo.png) Deploy a Production Ready Kubernetes Cluster ============================================ If you have questions, join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**. +You can get your invite [here](http://slack.k8s.io/) -- Can be deployed on **AWS, GCE, Azure, OpenStack or Baremetal** -- **High available** cluster +- Can be deployed on **AWS, GCE, Azure, OpenStack, vSphere, Oracle Cloud Infrastructure (Experimental), or Baremetal** +- **Highly available** cluster - **Composable** (Choice of the network plugin for instance) -- Support most popular **Linux distributions** +- Supports most popular **Linux distributions** - **Continuous integration tests** Quick Start @@ -18,8 +19,11 @@ To deploy the cluster you can use : ### Ansible + # Install dependencies from ``requirements.txt`` + sudo pip install -r requirements.txt + # Copy ``inventory/sample`` as ``inventory/mycluster`` - cp -rfp inventory/sample inventory/mycluster + cp -rfp inventory/sample/* inventory/mycluster # Update Ansible inventory file with inventory builder declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) @@ -32,9 +36,27 @@ To deploy the cluster you can use : # Deploy Kubespray with Ansible Playbook ansible-playbook -i inventory/mycluster/hosts.ini cluster.yml +Note: When Ansible is already installed via system packages on the control machine, other python packages installed via `sudo pip install -r requirements.txt` will go to a different directory tree (e.g. `/usr/local/lib/python2.7/dist-packages` on Ubuntu) from Ansible's (e.g. `/usr/lib/python2.7/dist-packages/ansible` still on Ubuntu). +As a consequence, `ansible-playbook` command will fail with: +``` +ERROR! no action detected in task. This often indicates a misspelled module name, or incorrect module path. +``` +probably pointing on a task depending on a module present in requirements.txt (i.e. "unseal vault"). + +One way of solving this would be to uninstall the Ansible package and then, to install it via pip but it is not always possible. +A workaround consists of setting `ANSIBLE_LIBRARY` and `ANSIBLE_MODULE_UTILS` environment variables respectively to the `ansible/modules` and `ansible/module_utils` subdirectories of pip packages installation location, which can be found in the Location field of the output of `pip show [package]` before executing `ansible-playbook`. + ### Vagrant - # Simply running `vagrant up` (for tests purposes) +For Vagrant we need to install python dependencies for provisioning tasks. +Check if Python and pip are installed: + + python -V && pip -V + +If this returns the version of the software, you're good to go. If not, download and install Python from here +Install the necessary requirements + + sudo pip install -r requirements.txt vagrant up Documents @@ -52,6 +74,7 @@ Documents - [Vagrant install](docs/vagrant.md) - [CoreOS bootstrap](docs/coreos.md) - [Debian Jessie setup](docs/debian.md) +- [openSUSE setup](docs/opensuse.md) - [Downloaded artifacts](docs/downloads.md) - [Cloud providers](docs/cloud.md) - [OpenStack](docs/openstack.md) @@ -66,28 +89,36 @@ Supported Linux Distributions ----------------------------- - **Container Linux by CoreOS** -- **Debian** Jessie -- **Ubuntu** 16.04 +- **Debian** Jessie, Stretch, Wheezy +- **Ubuntu** 16.04, 18.04 - **CentOS/RHEL** 7 - **Fedora/CentOS** Atomic +- **openSUSE** Leap 42.3/Tumbleweed Note: Upstart/SysV init based OS types are not supported. -Versions of supported components --------------------------------- - -- [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.9.3 -- [etcd](https://github.com/coreos/etcd/releases) v3.2.4 -- [flanneld](https://github.com/coreos/flannel/releases) v0.9.1 -- [calico](https://docs.projectcalico.org/v2.5/releases/) v2.5.0 -- [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) -- [cilium](https://github.com/cilium/cilium) v1.0.0-rc4 -- [contiv](https://github.com/contiv/install/releases) v1.0.3 -- [weave](http://weave.works/) v2.0.1 -- [docker](https://www.docker.com/) v17.03 (see note) -- [rkt](https://coreos.com/rkt/docs/latest/) v1.21.0 (see Note 2) - -Note: kubernetes doesn't support newer docker versions. Among other things kubelet currently breaks on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin). +Supported Components +-------------------- + +- Core + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.11.3 + - [etcd](https://github.com/coreos/etcd) v3.2.18 + - [docker](https://www.docker.com/) v17.03 (see note) + - [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2) +- Network Plugin + - [calico](https://github.com/projectcalico/calico) v3.1.3 + - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) + - [cilium](https://github.com/cilium/cilium) v1.2.0 + - [contiv](https://github.com/contiv/install) v1.1.7 + - [flanneld](https://github.com/coreos/flannel) v0.10.0 + - [weave](https://github.com/weaveworks/weave) v2.4.0 +- Application + - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11 + - [cert-manager](https://github.com/jetstack/cert-manager) v0.4.1 + - [coredns](https://github.com/coredns/coredns) v1.2.2 + - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.19.0 + +Note: kubernetes doesn't support newer docker versions ("Version 17.03 is recommended... Versions 17.06+ might work, but have not yet been tested and verified by the Kubernetes node team" cf. [Bootstrapping Clusters with kubeadm](https://kubernetes.io/docs/setup/independent/install-kubeadm/#installing-docker)). Among other things kubelet currently breaks on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin). Note 2: rkt support as docker alternative is limited to control plane (etcd and kubelet). Docker is still used for Kubernetes cluster workloads and network @@ -105,6 +136,9 @@ Requirements - **Your ssh key must be copied** to all the servers part of your inventory. - The **firewalls are not managed**, you'll need to implement your own rules the way you used to. in order to avoid any issue during deployment you should disable your firewall. +- If kubespray is ran from non-root user account, correct privilege escalation method + should be configured in the target servers. Then the `ansible_become` flag + or command parameters `--become or -b` should be specified. Network Plugins --------------- @@ -117,7 +151,7 @@ You can choose between 6 network plugins. (default: `calico`, except Vagrant use - [canal](https://github.com/projectcalico/canal): a composition of calico and flannel plugins. -- [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic. +- [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic. - [contiv](docs/contiv.md): supports vlan, vxlan, bgp and Cisco SDN networking. This plugin is able to apply firewall policies, segregate containers in multiple network and bridging pods onto physical networks. @@ -147,8 +181,6 @@ Tools and projects on top of Kubespray CI Tests -------- -![Gitlab Logo](https://s27.postimg.org/wmtaig1wz/gitlabci.png) - [![Build graphs](https://gitlab.com/kubespray-ci/kubernetes-incubator__kubespray/badges/master/build.svg)](https://gitlab.com/kubespray-ci/kubernetes-incubator__kubespray/pipelines) CI/end-to-end tests sponsored by Google (GCE) diff --git a/SECURITY_CONTACTS b/SECURITY_CONTACTS new file mode 100644 index 00000000000..df77f138252 --- /dev/null +++ b/SECURITY_CONTACTS @@ -0,0 +1,13 @@ +# Defined below are the security contacts for this repo. +# +# They are the contact point for the Product Security Team to reach out +# to for triaging and handling of incoming issues. +# +# The below names agree to abide by the +# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) +# and will be removed and replaced if they violate that agreement. +# +# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE +# INSTRUCTIONS AT https://kubernetes.io/security/ +atoms +mattymo \ No newline at end of file diff --git a/Vagrantfile b/Vagrantfile index 9db4be3a1b9..df650f1e8cd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -3,7 +3,7 @@ require 'fileutils' -Vagrant.require_version ">= 1.9.0" +Vagrant.require_version ">= 2.0.0" CONFIG = File.join(File.dirname(__FILE__), "vagrant/config.rb") @@ -18,6 +18,8 @@ SUPPORTED_OS = { "coreos-beta" => {box: "coreos-beta", bootstrap_os: "coreos", user: "core", box_url: COREOS_URL_TEMPLATE % ["beta"]}, "ubuntu" => {box: "bento/ubuntu-16.04", bootstrap_os: "ubuntu", user: "vagrant"}, "centos" => {box: "centos/7", bootstrap_os: "centos", user: "vagrant"}, + "opensuse" => {box: "opensuse/openSUSE-42.3-x86_64", bootstrap_os: "opensuse", use: "vagrant"}, + "opensuse-tumbleweed" => {box: "opensuse/openSUSE-Tumbleweed-x86_64", bootstrap_os: "opensuse", use: "vagrant"}, } # Defaults for config options defined in CONFIG @@ -42,6 +44,8 @@ $kube_node_instances_with_disks = false $kube_node_instances_with_disks_size = "20G" $kube_node_instances_with_disks_number = 2 +$playbook = "cluster.yml" + $local_release_dir = "/vagrant/temp" host_vars = {} @@ -52,7 +56,7 @@ end $box = SUPPORTED_OS[$os][:box] # if $inventory is not set, try to use example -$inventory = File.join(File.dirname(__FILE__), "inventory") if ! $inventory +$inventory = File.join(File.dirname(__FILE__), "inventory", "sample") if ! $inventory # if $inventory has a hosts file use it, otherwise copy over vars etc # to where vagrant expects dynamic inventory to be. @@ -84,7 +88,6 @@ Vagrant.configure("2") do |config| if Vagrant.has_plugin?("vagrant-vbguest") then config.vbguest.auto_update = false end - (1..$num_instances).each do |i| config.vm.define vm_name = "%s-%02d" % [$instance_name_prefix, i] do |config| config.vm.hostname = vm_name @@ -110,8 +113,10 @@ Vagrant.configure("2") do |config| end end + config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] + $shared_folders.each do |src, dst| - config.vm.synced_folder src, dst + config.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] end config.vm.provider :virtualbox do |vb| @@ -135,12 +140,6 @@ Vagrant.configure("2") do |config| config.vm.network :private_network, ip: ip - # workaround for Vagrant 1.9.1 and centos vm - # https://github.com/hashicorp/vagrant/issues/8096 - if Vagrant::VERSION == "1.9.1" && $os == "centos" - config.vm.provision "shell", inline: "service network restart", run: "always" - end - # Disable swap for each vm config.vm.provision "shell", inline: "swapoff -a" @@ -160,11 +159,11 @@ Vagrant.configure("2") do |config| # when all the machines are up and ready. if i == $num_instances config.vm.provision "ansible" do |ansible| - ansible.playbook = "cluster.yml" + ansible.playbook = $playbook if File.exist?(File.join(File.dirname($inventory), "hosts")) ansible.inventory_path = $inventory end - ansible.sudo = true + ansible.become = true ansible.limit = "all" ansible.host_key_checking = false ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache"] diff --git a/ansible.cfg b/ansible.cfg index 732e3bf6e31..6912e4d0891 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -12,3 +12,6 @@ library = ./library callback_whitelist = profile_tasks roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles deprecation_warnings=False +inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds +[inventory] +ignore_patterns = artifacts, credentials diff --git a/cluster.yml b/cluster.yml index 00c68a5939f..6a9de14da1a 100644 --- a/cluster.yml +++ b/cluster.yml @@ -26,18 +26,19 @@ setup: delegate_to: "{{item}}" delegate_facts: True - with_items: "{{ groups['k8s-cluster'] + groups['etcd'] }}" + with_items: "{{ groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]) }}" - hosts: k8s-cluster:etcd:calico-rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray-defaults} - { role: kubernetes/preinstall, tags: preinstall } - - { role: docker, tags: docker } + - { role: docker, tags: docker, when: container_manager == 'docker' } + - { role: cri-o, tags: crio, when: container_manager == 'crio' } - role: rkt tags: rkt when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]" - - { role: download, tags: download, skip_downloads: false } + - { role: download, tags: download, when: "not skip_downloads" } environment: "{{proxy_env}}" - hosts: etcd:k8s-cluster:vault:calico-rr @@ -51,13 +52,13 @@ any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray-defaults} - - { role: etcd, tags: etcd, etcd_cluster_setup: true } + - { role: etcd, tags: etcd, etcd_cluster_setup: true, etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" } - hosts: k8s-cluster:calico-rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray-defaults} - - { role: etcd, tags: etcd, etcd_cluster_setup: false } + - { role: etcd, tags: etcd, etcd_cluster_setup: false, etcd_events_cluster_setup: false } - hosts: etcd:k8s-cluster:vault:calico-rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" @@ -93,6 +94,7 @@ roles: - { role: kubespray-defaults} - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" } + - { role: win_nodes/kubernetes_patch, tags: win_nodes, when: "kubeadm_enabled" } - hosts: kube-master any_errors_fatal: "{{ any_errors_fatal | default(true) }}" diff --git a/contrib/azurerm/README.md b/contrib/azurerm/README.md index c15d3ecf253..b83aeeb9bbf 100644 --- a/contrib/azurerm/README.md +++ b/contrib/azurerm/README.md @@ -9,8 +9,8 @@ Resource Group. It will not install Kubernetes itself, this has to be done in a ## Requirements -- [Install azure-cli](https://docs.microsoft.com/en-us/azure/xplat-cli-install) -- [Login with azure-cli](https://docs.microsoft.com/en-us/azure/xplat-cli-connect) +- [Install azure-cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) +- [Login with azure-cli](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest) - Dedicated Resource Group created in the Azure Portal or through azure-cli ## Configuration through group_vars/all diff --git a/contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2 b/contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2 index 696be6d57e8..3e9728e715a 100644 --- a/contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2 +++ b/contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2 @@ -1,6 +1,6 @@ {% for vm in vm_ip_list %} -{% if not use_bastion or vm.virtualMachinename == 'bastion' %} +{% if not use_bastion or vm.virtualMachine.name == 'bastion' %} {{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.publicIpAddresses[0].ipAddress }} ip={{ vm.virtualMachine.network.privateIpAddresses[0] }} {% else %} {{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.privateIpAddresses[0] }} diff --git a/contrib/inventory_builder/inventory.py b/contrib/inventory_builder/inventory.py index 94e5c7c352c..3ed14cbe394 100644 --- a/contrib/inventory_builder/inventory.py +++ b/contrib/inventory_builder/inventory.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/contrib/metallb/README.md b/contrib/metallb/README.md new file mode 100644 index 00000000000..7095d58f816 --- /dev/null +++ b/contrib/metallb/README.md @@ -0,0 +1,10 @@ +# Deploy MetalLB into Kubespray/Kubernetes +``` +MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation. In short, it allows you to create Kubernetes services of type “LoadBalancer” in clusters that don’t run on a cloud provider, and thus cannot simply hook into paid products to provide load-balancers. +``` +This playbook aims to automate [this](https://metallb.universe.tf/tutorial/layer2/tutorial). It deploys MetalLB into kubernetes and sets up a layer 2 loadbalancer. + +## Install +``` +ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/metallb/metallb.yml +``` diff --git a/contrib/metallb/metallb.yml b/contrib/metallb/metallb.yml new file mode 100644 index 00000000000..618a1d22315 --- /dev/null +++ b/contrib/metallb/metallb.yml @@ -0,0 +1,6 @@ +--- +- hosts: kube-master[0] + tags: + - "provision" + roles: + - { role: provision } diff --git a/contrib/metallb/roles/provision/defaults/main.yml b/contrib/metallb/roles/provision/defaults/main.yml new file mode 100644 index 00000000000..c7d0bf904b3 --- /dev/null +++ b/contrib/metallb/roles/provision/defaults/main.yml @@ -0,0 +1,7 @@ +--- +metallb: + ip_range: "10.5.0.50-10.5.0.99" + limits: + cpu: "100m" + memory: "100Mi" + port: "7472" diff --git a/contrib/metallb/roles/provision/tasks/main.yml b/contrib/metallb/roles/provision/tasks/main.yml new file mode 100644 index 00000000000..47c4b624a8d --- /dev/null +++ b/contrib/metallb/roles/provision/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: "Kubernetes Apps | Lay Down MetalLB" + become: true + template: { src: "{{ item }}.j2", dest: "{{ kube_config_dir }}/{{ item }}" } + with_items: ["metallb.yml", "metallb-config.yml"] + register: "rendering" + when: + - "inventory_hostname == groups['kube-master'][0]" +- name: "Kubernetes Apps | Install and configure MetalLB" + kube: + name: "MetalLB" + filename: "{{ kube_config_dir }}/metallb.yml" + state: "{{ item.changed | ternary('latest','present') }}" + with_items: "{{ rendering.results }}" + when: + - "inventory_hostname == groups['kube-master'][0]" diff --git a/contrib/metallb/roles/provision/templates/metallb-config.yml.j2 b/contrib/metallb/roles/provision/templates/metallb-config.yml.j2 new file mode 100644 index 00000000000..2e58f2d5f15 --- /dev/null +++ b/contrib/metallb/roles/provision/templates/metallb-config.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: loadbalanced + protocol: layer2 + addresses: + - {{ metallb.ip_range }} diff --git a/contrib/metallb/roles/provision/templates/metallb.yml.j2 b/contrib/metallb/roles/provision/templates/metallb.yml.j2 new file mode 100644 index 00000000000..c9532f014ae --- /dev/null +++ b/contrib/metallb/roles/provision/templates/metallb.yml.j2 @@ -0,0 +1,254 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: metallb-system + labels: + app: metallb +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: metallb-system + name: controller + labels: + app: metallb +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: metallb-system + name: speaker + labels: + app: metallb + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metallb-system:controller + labels: + app: metallb +rules: +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metallb-system:speaker + labels: + app: metallb +rules: +- apiGroups: [""] + resources: ["services", "endpoints", "nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: metallb-system + name: leader-election + labels: + app: metallb +rules: +- apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["metallb-speaker"] + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: metallb-system + name: config-watcher + labels: + app: metallb +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create"] +--- + +## Role bindings +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metallb-system:controller + labels: + app: metallb +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metallb-system:speaker + labels: + app: metallb +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: metallb-system + name: config-watcher + labels: + app: metallb +subjects: +- kind: ServiceAccount + name: controller +- kind: ServiceAccount + name: speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: config-watcher +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: metallb-system + name: leader-election + labels: + app: metallb +subjects: +- kind: ServiceAccount + name: speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election +--- +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + namespace: metallb-system + name: speaker + labels: + app: metallb + component: speaker +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + labels: + app: metallb + component: speaker + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ metallb.port }}" + spec: + serviceAccountName: speaker + terminationGracePeriodSeconds: 0 + hostNetwork: true + containers: + - name: speaker + image: metallb/speaker:v0.6.2 + imagePullPolicy: IfNotPresent + args: + - --port={{ metallb.port }} + - --config=config + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: monitoring + containerPort: {{ metallb.port }} + resources: + limits: + cpu: {{ metallb.limits.cpu }} + memory: {{ metallb.limits.memory }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - all + add: + - net_raw + +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + namespace: metallb-system + name: controller + labels: + app: metallb + component: controller +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + labels: + app: metallb + component: controller + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ metallb.port }}" + spec: + serviceAccountName: controller + terminationGracePeriodSeconds: 0 + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + containers: + - name: controller + image: metallb/controller:v0.6.2 + imagePullPolicy: IfNotPresent + args: + - --port={{ metallb.port }} + - --config=config + ports: + - name: monitoring + containerPort: {{ metallb.port }} + resources: + limits: + cpu: {{ metallb.limits.cpu }} + memory: {{ metallb.limits.memory }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + +--- + + diff --git a/contrib/network-storage/glusterfs/group_vars b/contrib/network-storage/glusterfs/group_vars index d64da8dc611..6a3f85e47c9 120000 --- a/contrib/network-storage/glusterfs/group_vars +++ b/contrib/network-storage/glusterfs/group_vars @@ -1 +1 @@ -../../../inventory/group_vars \ No newline at end of file +../../../inventory/local/group_vars \ No newline at end of file diff --git a/contrib/network-storage/glusterfs/inventory.example b/contrib/network-storage/glusterfs/inventory.example index 41e36c8da3c..15fbad0a815 100644 --- a/contrib/network-storage/glusterfs/inventory.example +++ b/contrib/network-storage/glusterfs/inventory.example @@ -12,7 +12,7 @@ # ## As in the previous case, you can set ip to give direct communication on internal IPs # gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc ip=10.3.0.7 # gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc ip=10.3.0.8 -# gfs_node1 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc ip=10.3.0.9 +# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc ip=10.3.0.9 # [kube-master] # node1 diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml b/contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml index 5ca49386706..b9f0d2d1d3b 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml @@ -2,7 +2,7 @@ # For Ubuntu. glusterfs_default_release: "" glusterfs_ppa_use: yes -glusterfs_ppa_version: "3.8" +glusterfs_ppa_version: "4.1" # Gluster configuration. gluster_mount_dir: /mnt/gluster diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml b/contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml index 1c876338863..ef9a71eba40 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml @@ -2,7 +2,7 @@ # For Ubuntu. glusterfs_default_release: "" glusterfs_ppa_use: yes -glusterfs_ppa_version: "3.8" +glusterfs_ppa_version: "3.12" # Gluster configuration. gluster_mount_dir: /mnt/gluster diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml b/contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml index 13c595f745c..e931068ae37 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml @@ -1,2 +1,2 @@ --- -glusterfs_daemon: glusterfs-server +glusterfs_daemon: glusterd diff --git a/contrib/network-storage/heketi/README.md b/contrib/network-storage/heketi/README.md new file mode 100644 index 00000000000..aa1b656e5c4 --- /dev/null +++ b/contrib/network-storage/heketi/README.md @@ -0,0 +1,16 @@ +# Deploy Heketi/Glusterfs into Kubespray/Kubernetes +This playbook aims to automate [this](https://github.com/heketi/heketi/blob/master/docs/admin/install-kubernetes.md) tutorial. It deploys heketi/glusterfs into kubernetes and sets up a storageclass. + +## Client Setup +Heketi provides a CLI that provides users with a means to administer the deployment and configuration of GlusterFS in Kubernetes. [Download and install the heketi-cli](https://github.com/heketi/heketi/releases) on your client machine. + +## Install +Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup. +``` +ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml +``` + +## Tear down +``` +ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi-tear-down.yml +``` diff --git a/contrib/network-storage/heketi/heketi-tear-down.yml b/contrib/network-storage/heketi/heketi-tear-down.yml new file mode 100644 index 00000000000..92b9f92d64e --- /dev/null +++ b/contrib/network-storage/heketi/heketi-tear-down.yml @@ -0,0 +1,9 @@ +--- +- hosts: kube-master[0] + roles: + - { role: tear-down } + +- hosts: heketi-node + become: yes + roles: + - { role: tear-down-disks } diff --git a/contrib/network-storage/heketi/heketi.yml b/contrib/network-storage/heketi/heketi.yml new file mode 100644 index 00000000000..3ec719e95b7 --- /dev/null +++ b/contrib/network-storage/heketi/heketi.yml @@ -0,0 +1,10 @@ +--- +- hosts: heketi-node + roles: + - { role: prepare } + +- hosts: kube-master[0] + tags: + - "provision" + roles: + - { role: provision } diff --git a/contrib/network-storage/heketi/inventory.yml.sample b/contrib/network-storage/heketi/inventory.yml.sample new file mode 100644 index 00000000000..7d488d1ba14 --- /dev/null +++ b/contrib/network-storage/heketi/inventory.yml.sample @@ -0,0 +1,26 @@ +all: + vars: + heketi_admin_key: "11elfeinhundertundelf" + heketi_user_key: "!!einseinseins" + children: + k8s-cluster: + vars: + kubelet_fail_swap_on: false + children: + kube-master: + hosts: + node1: + etcd: + hosts: + node2: + kube-node: + hosts: &kube_nodes + node1: + node2: + node3: + node4: + heketi-node: + vars: + disk_volume_device_1: "/dev/vdb" + hosts: + <<: *kube_nodes diff --git a/contrib/network-storage/heketi/requirements.txt b/contrib/network-storage/heketi/requirements.txt new file mode 100644 index 00000000000..45c1e038e5f --- /dev/null +++ b/contrib/network-storage/heketi/requirements.txt @@ -0,0 +1 @@ +jmespath diff --git a/contrib/network-storage/heketi/roles/prepare/tasks/main.yml b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml new file mode 100644 index 00000000000..e4db23365bf --- /dev/null +++ b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- name: "Load lvm kernel modules" + become: true + with_items: + - "dm_snapshot" + - "dm_mirror" + - "dm_thin_pool" + modprobe: + name: "{{ item }}" + state: "present" + +- name: "Install glusterfs mount utils (RedHat)" + become: true + yum: + name: "glusterfs-fuse" + state: "present" + when: "ansible_os_family == 'RedHat'" + +- name: "Install glusterfs mount utils (Debian)" + become: true + apt: + name: "glusterfs-client" + state: "present" + when: "ansible_os_family == 'Debian'" diff --git a/contrib/network-storage/heketi/roles/provision/defaults/main.yml b/contrib/network-storage/heketi/roles/provision/defaults/main.yml new file mode 100644 index 00000000000..ed97d539c09 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/defaults/main.yml @@ -0,0 +1 @@ +--- diff --git a/contrib/network-storage/heketi/roles/provision/handlers/main.yml b/contrib/network-storage/heketi/roles/provision/handlers/main.yml new file mode 100644 index 00000000000..9e876de177b --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: "stop port forwarding" + command: "killall " diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml new file mode 100644 index 00000000000..572913a63e0 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml @@ -0,0 +1,56 @@ +# Bootstrap heketi +- name: "Get state of heketi service, deployment and pods." + register: "initial_heketi_state" + changed_when: false + command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" +- name: "Bootstrap heketi." + when: + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" + include_tasks: "bootstrap/deploy.yml" + +# Prepare heketi topology +- name: "Get heketi initial pod state." + register: "initial_heketi_pod" + command: "{{ bin_dir }}/kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" + changed_when: false +- name: "Ensure heketi bootstrap pod is up." + assert: + that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1" +- set_fact: + initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}" +- name: "Test heketi topology." + changed_when: false + register: "heketi_topology" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" +- name: "Load heketi topology." + when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" + include_tasks: "bootstrap/topology.yml" + +# Provision heketi database volume +- name: "Prepare heketi volumes." + include_tasks: "bootstrap/volumes.yml" + +# Remove bootstrap heketi +- name: "Tear down bootstrap." + include_tasks: "bootstrap/tear-down.yml" + +# Prepare heketi storage +- name: "Test heketi storage." + command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" + changed_when: false + register: "heketi_storage_state" +# ensure endpoints actually exist before trying to move database data to it +- name: "Create heketi storage." + include_tasks: "bootstrap/storage.yml" + vars: + secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" + endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" + service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" + when: + - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml new file mode 100644 index 00000000000..3037d8b77ca --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml @@ -0,0 +1,24 @@ +--- +- name: "Kubernetes Apps | Lay Down Heketi Bootstrap" + become: true + template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" } + register: "rendering" +- name: "Kubernetes Apps | Install and configure Heketi Bootstrap" + kube: + name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/heketi-bootstrap.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" +- name: "Wait for heketi bootstrap to complete." + changed_when: false + register: "initial_heketi_state" + vars: + initial_heketi_state: { stdout: "{}" } + pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" + deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" + command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json" + until: + - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" + - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml new file mode 100644 index 00000000000..be3c42cafa7 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml @@ -0,0 +1,33 @@ +--- +- name: "Test heketi storage." + command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json" + changed_when: false + register: "heketi_storage_state" +- name: "Create heketi storage." + kube: + name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/heketi-storage-bootstrap.json" + state: "present" + vars: + secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" + endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" + service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" + when: + - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" + register: "heketi_storage_result" +- name: "Get state of heketi database copy job." + command: "{{ bin_dir }}/kubectl get jobs --output=json" + changed_when: false + register: "heketi_storage_state" + vars: + heketi_storage_state: { stdout: "{}" } + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]" + until: + - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1" + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml new file mode 100644 index 00000000000..0ffd6f469f2 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml @@ -0,0 +1,14 @@ +--- +- name: "Get existing Heketi deploy resources." + command: "{{ bin_dir }}/kubectl get all --selector=\"deploy-heketi\" -o=json" + register: "heketi_resources" + changed_when: false +- name: "Delete bootstrap Heketi." + command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\"" + when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0" +- name: "Ensure there is nothing left over." + command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json" + register: "heketi_result" + until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml new file mode 100644 index 00000000000..7d2c5981e7e --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml @@ -0,0 +1,26 @@ +--- +- name: "Get heketi topology." + changed_when: false + register: "heketi_topology" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" +- name: "Render heketi topology template." + become: true + vars: { nodes: "{{ groups['heketi-node'] }}" } + register: "render" + template: + src: "topology.json.j2" + dest: "{{ kube_config_dir }}/topology.json" +- name: "Copy topology configuration into container." + changed_when: false + command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json" +- name: "Load heketi topology." + when: "render.changed" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json" + register: "load_heketi" +- name: "Get heketi topology." + changed_when: false + register: "heketi_topology" + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" + until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml new file mode 100644 index 00000000000..d5da1a12588 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml @@ -0,0 +1,41 @@ +--- +- name: "Get heketi volume ids." + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json" + changed_when: false + register: "heketi_volumes" +- name: "Get heketi volumes." + changed_when: false + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json" + with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" + loop_control: { loop_var: "volume_id" } + register: "volumes_information" +- name: "Test heketi database volume." + set_fact: { heketi_database_volume_exists: true } + with_items: "{{ volumes_information.results }}" + loop_control: { loop_var: "volume_information" } + vars: { volume: "{{ volume_information.stdout|from_json }}" } + when: "volume.name == 'heketidbstorage'" +- name: "Provision database volume." + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} setup-openshift-heketi-storage" + when: "heketi_database_volume_exists is undefined" +- name: "Copy configuration from pod." + become: true + command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage-bootstrap.json" +- name: "Get heketi volume ids." + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json" + changed_when: false + register: "heketi_volumes" +- name: "Get heketi volumes." + changed_when: false + command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json" + with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" + loop_control: { loop_var: "volume_id" } + register: "volumes_information" +- name: "Test heketi database volume." + set_fact: { heketi_database_volume_created: true } + with_items: "{{ volumes_information.results }}" + loop_control: { loop_var: "volume_information" } + vars: { volume: "{{ volume_information.stdout|from_json }}" } + when: "volume.name == 'heketidbstorage'" +- name: "Ensure heketi database volume exists." + assert: { that: "heketi_database_volume_created is defined" , msg: "Heketi database volume does not exist." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml b/contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml new file mode 100644 index 00000000000..238f29bc214 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml @@ -0,0 +1,4 @@ +--- +- name: "Clean up left over jobs." + command: "{{ bin_dir }}/kubectl delete jobs,pods --selector=\"deploy-heketi\"" + changed_when: false diff --git a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml new file mode 100644 index 00000000000..e4615996937 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml @@ -0,0 +1,38 @@ +--- +- name: "Kubernetes Apps | Lay Down GlusterFS Daemonset" + template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" } + become: true + register: "rendering" +- name: "Kubernetes Apps | Install and configure GlusterFS daemonset" + kube: + name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/glusterfs-daemonset.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" +- name: "Kubernetes Apps | Label GlusterFS nodes" + include_tasks: "glusterfs/label.yml" + with_items: "{{ groups['heketi-node'] }}" + loop_control: + loop_var: "node" +- name: "Kubernetes Apps | Wait for daemonset to become available." + register: "daemonset_state" + command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true" + changed_when: false + vars: + daemonset_state: { stdout: "{}" } + ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}" + desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}" + until: "ready >= 3" + retries: 60 + delay: 5 + +- name: "Kubernetes Apps | Lay Down Heketi Service Account" + template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" } + become: true + register: "rendering" +- name: "Kubernetes Apps | Install and configure Heketi Service Account" + kube: + name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/heketi-service-account.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml new file mode 100644 index 00000000000..61729a5e29b --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml @@ -0,0 +1,11 @@ +--- +- register: "label_present" + command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" + changed_when: false +- name: "Assign storage label" + when: "label_present.stdout_lines|length == 0" + command: "{{ bin_dir }}/kubectl label node {{ node }} storagenode=glusterfs" +- register: "label_present" + command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" + changed_when: false +- assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml new file mode 100644 index 00000000000..029baef947b --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml @@ -0,0 +1,26 @@ +--- +- name: "Kubernetes Apps | Lay Down Heketi" + become: true + template: { src: "heketi-deployment.json.j2", dest: "{{ kube_config_dir }}/heketi-deployment.json" } + register: "rendering" +- name: "Kubernetes Apps | Install and configure Heketi" + kube: + name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/heketi-deployment.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" +- name: "Ensure heketi is up and running." + changed_when: false + register: "heketi_state" + vars: + heketi_state: { stdout: "{}" } + pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" + deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" + command: "{{ bin_dir }}/kubectl get deployments,pods --selector=glusterfs --output=json" + until: + - "heketi_state.stdout|from_json|json_query(pods_query) == 'True'" + - "heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" + retries: 60 + delay: 5 +- set_fact: + heketi_pod_name: "{{ heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.name|[0]\") }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml new file mode 100644 index 00000000000..23a2b4f9c72 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: "Kubernetes Apps | GlusterFS" + include_tasks: "glusterfs.yml" + +- name: "Kubernetes Apps | Heketi Secrets" + include_tasks: "secret.yml" + +- name: "Kubernetes Apps | Test Heketi" + register: "heketi_service_state" + command: "{{bin_dir}}/kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true" + changed_when: false + +- name: "Kubernetes Apps | Bootstrap Heketi" + when: "heketi_service_state.stdout == \"\"" + include_tasks: "bootstrap.yml" + +- name: "Kubernetes Apps | Heketi" + include_tasks: "heketi.yml" + +- name: "Kubernetes Apps | Heketi Topology" + include_tasks: "topology.yml" + +- name: "Kubernetes Apps | Heketi Storage" + include_tasks: "storage.yml" + +- name: "Kubernetes Apps | Storage Class" + include_tasks: "storageclass.yml" + +- name: "Clean up" + include_tasks: "cleanup.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml new file mode 100644 index 00000000000..8ca21bcb693 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml @@ -0,0 +1,27 @@ +--- +- register: "clusterrolebinding_state" + command: "{{bin_dir}}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + changed_when: false +- name: "Kubernetes Apps | Deploy cluster role binding." + when: "clusterrolebinding_state.stdout == \"\"" + command: "{{bin_dir}}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" +- register: "clusterrolebinding_state" + command: "{{bin_dir}}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } + +- register: "secret_state" + command: "{{bin_dir}}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + changed_when: false +- name: "Render Heketi secret configuration." + become: true + template: + src: "heketi.json.j2" + dest: "{{ kube_config_dir }}/heketi.json" +- name: "Deploy Heketi config secret" + when: "secret_state.stdout == \"\"" + command: "{{bin_dir}}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" +- register: "secret_state" + command: "{{bin_dir}}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/storage.yml new file mode 100644 index 00000000000..881084bbe1d --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/storage.yml @@ -0,0 +1,11 @@ +--- +- name: "Kubernetes Apps | Lay Down Heketi Storage" + become: true + vars: { nodes: "{{ groups['heketi-node'] }}" } + template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" } + register: "rendering" +- name: "Kubernetes Apps | Install and configure Heketi Storage" + kube: + name: "GlusterFS" + filename: "{{ kube_config_dir }}/heketi-storage.json" + state: "{{ rendering.changed | ternary('latest', 'present') }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml new file mode 100644 index 00000000000..afd818eb31b --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml @@ -0,0 +1,25 @@ +--- +- name: "Test storage class." + command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json" + register: "storageclass" + changed_when: false +- name: "Test heketi service." + command: "{{ bin_dir }}/kubectl get service heketi --ignore-not-found=true --output=json" + register: "heketi_service" + changed_when: false +- name: "Ensure heketi service is available." + assert: { that: "heketi_service.stdout != \"\"" } +- name: "Render storage class configuration." + become: true + vars: + endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}" + template: + src: "storageclass.yml.j2" + dest: "{{ kube_config_dir }}/storageclass.yml" + register: "rendering" +- name: "Kubernetes Apps | Install and configure Storace Class" + kube: + name: "GlusterFS" + kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/storageclass.yml" + state: "{{ rendering.changed | ternary('latest', 'present') }}" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml new file mode 100644 index 00000000000..dd1e272beb0 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml @@ -0,0 +1,25 @@ +--- +- name: "Get heketi topology." + register: "heketi_topology" + changed_when: false + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" +- name: "Render heketi topology template." + become: true + vars: { nodes: "{{ groups['heketi-node'] }}" } + register: "rendering" + template: + src: "topology.json.j2" + dest: "{{ kube_config_dir }}/topology.json" +- name: "Copy topology configuration into container." + when: "rendering.changed" + command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json" +- name: "Load heketi topology." + when: "rendering.changed" + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json" +- name: "Get heketi topology." + register: "heketi_topology" + changed_when: false + command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json" + until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length" + retries: 60 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 new file mode 100644 index 00000000000..eddd57eb817 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 @@ -0,0 +1,144 @@ +{ + "kind": "DaemonSet", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "glusterfs", + "labels": { + "glusterfs": "deployment" + }, + "annotations": { + "description": "GlusterFS Daemon Set", + "tags": "glusterfs" + } + }, + "spec": { + "template": { + "metadata": { + "name": "glusterfs", + "labels": { + "glusterfs-node": "daemonset" + } + }, + "spec": { + "nodeSelector": { + "storagenode" : "glusterfs" + }, + "hostNetwork": true, + "containers": [ + { + "image": "gluster/gluster-centos:gluster4u0_centos7", + "imagePullPolicy": "IfNotPresent", + "name": "glusterfs", + "volumeMounts": [ + { + "name": "glusterfs-heketi", + "mountPath": "/var/lib/heketi" + }, + { + "name": "glusterfs-run", + "mountPath": "/run" + }, + { + "name": "glusterfs-lvm", + "mountPath": "/run/lvm" + }, + { + "name": "glusterfs-etc", + "mountPath": "/etc/glusterfs" + }, + { + "name": "glusterfs-logs", + "mountPath": "/var/log/glusterfs" + }, + { + "name": "glusterfs-config", + "mountPath": "/var/lib/glusterd" + }, + { + "name": "glusterfs-dev", + "mountPath": "/dev" + }, + { + "name": "glusterfs-cgroup", + "mountPath": "/sys/fs/cgroup" + } + ], + "securityContext": { + "capabilities": {}, + "privileged": true + }, + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 60, + "exec": { + "command": [ + "/bin/bash", + "-c", + "systemctl status glusterd.service" + ] + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 60, + "exec": { + "command": [ + "/bin/bash", + "-c", + "systemctl status glusterd.service" + ] + } + } + } + ], + "volumes": [ + { + "name": "glusterfs-heketi", + "hostPath": { + "path": "/var/lib/heketi" + } + }, + { + "name": "glusterfs-run" + }, + { + "name": "glusterfs-lvm", + "hostPath": { + "path": "/run/lvm" + } + }, + { + "name": "glusterfs-etc", + "hostPath": { + "path": "/etc/glusterfs" + } + }, + { + "name": "glusterfs-logs", + "hostPath": { + "path": "/var/log/glusterfs" + } + }, + { + "name": "glusterfs-config", + "hostPath": { + "path": "/var/lib/glusterd" + } + }, + { + "name": "glusterfs-dev", + "hostPath": { + "path": "/dev" + } + }, + { + "name": "glusterfs-cgroup", + "hostPath": { + "path": "/sys/fs/cgroup" + } + } + ] + } + } + } +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2 new file mode 100644 index 00000000000..bdcf3e9588f --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2 @@ -0,0 +1,133 @@ +{ + "kind": "List", + "apiVersion": "v1", + "items": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "deploy-heketi", + "labels": { + "glusterfs": "heketi-service", + "deploy-heketi": "support" + }, + "annotations": { + "description": "Exposes Heketi Service" + } + }, + "spec": { + "selector": { + "name": "deploy-heketi" + }, + "ports": [ + { + "name": "deploy-heketi", + "port": 8080, + "targetPort": 8080 + } + ] + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "deploy-heketi", + "labels": { + "glusterfs": "heketi-deployment", + "deploy-heketi": "deployment" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "name": "deploy-heketi", + "labels": { + "name": "deploy-heketi", + "glusterfs": "heketi-pod", + "deploy-heketi": "pod" + } + }, + "spec": { + "serviceAccountName": "heketi-service-account", + "containers": [ + { + "image": "heketi/heketi:7", + "imagePullPolicy": "Always", + "name": "deploy-heketi", + "env": [ + { + "name": "HEKETI_EXECUTOR", + "value": "kubernetes" + }, + { + "name": "HEKETI_DB_PATH", + "value": "/var/lib/heketi/heketi.db" + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + }, + { + "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", + "value": "y" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "name": "db", + "mountPath": "/var/lib/heketi" + }, + { + "name": "config", + "mountPath": "/etc/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "db" + }, + { + "name": "config", + "secret": { + "secretName": "heketi-config-secret" + } + } + ] + } + } + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2 new file mode 100644 index 00000000000..5eb71718cc9 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2 @@ -0,0 +1,159 @@ +{ + "kind": "List", + "apiVersion": "v1", + "items": [ + { + "kind": "Secret", + "apiVersion": "v1", + "metadata": { + "name": "heketi-db-backup", + "labels": { + "glusterfs": "heketi-db", + "heketi": "db" + } + }, + "data": { + }, + "type": "Opaque" + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-service", + "deploy-heketi": "support" + }, + "annotations": { + "description": "Exposes Heketi Service" + } + }, + "spec": { + "selector": { + "name": "heketi" + }, + "ports": [ + { + "name": "heketi", + "port": 8080, + "targetPort": 8080 + } + ] + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-deployment" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "name": "heketi", + "labels": { + "name": "heketi", + "glusterfs": "heketi-pod" + } + }, + "spec": { + "serviceAccountName": "heketi-service-account", + "containers": [ + { + "image": "heketi/heketi:7", + "imagePullPolicy": "Always", + "name": "heketi", + "env": [ + { + "name": "HEKETI_EXECUTOR", + "value": "kubernetes" + }, + { + "name": "HEKETI_DB_PATH", + "value": "/var/lib/heketi/heketi.db" + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + }, + { + "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", + "value": "y" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "mountPath": "/backupdb", + "name": "heketi-db-secret" + }, + { + "name": "db", + "mountPath": "/var/lib/heketi" + }, + { + "name": "config", + "mountPath": "/etc/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "db", + "glusterfs": { + "endpoints": "heketi-storage-endpoints", + "path": "heketidbstorage" + } + }, + { + "name": "heketi-db-secret", + "secret": { + "secretName": "heketi-db-backup" + } + }, + { + "name": "config", + "secret": { + "secretName": "heketi-config-secret" + } + } + ] + } + } + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2 new file mode 100644 index 00000000000..1dbcb9e962c --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2 @@ -0,0 +1,7 @@ +{ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": "heketi-service-account" + } +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2 new file mode 100644 index 00000000000..3089256c96f --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2 @@ -0,0 +1,54 @@ +{ + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "kind": "Endpoints", + "apiVersion": "v1", + "metadata": { + "name": "heketi-storage-endpoints", + "creationTimestamp": null + }, + "subsets": [ +{% set nodeblocks = [] %} +{% for node in nodes %} +{% set nodeblock %} + { + "addresses": [ + { + "ip": "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}" + } + ], + "ports": [ + { + "port": 1 + } + ] + } +{% endset %} +{% if nodeblocks.append(nodeblock) %}{% endif %} +{% endfor %} +{{ nodeblocks|join(',') }} + ] + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "heketi-storage-endpoints", + "creationTimestamp": null + }, + "spec": { + "ports": [ + { + "port": 1, + "targetPort": 0 + } + ] + }, + "status": { + "loadBalancer": {} + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 new file mode 100644 index 00000000000..5861b684b43 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 @@ -0,0 +1,44 @@ +{ + "_port_comment": "Heketi Server Port Number", + "port": "8080", + + "_use_auth": "Enable JWT authorization. Please enable for deployment", + "use_auth": true, + + "_jwt": "Private keys for access", + "jwt": { + "_admin": "Admin has access to all APIs", + "admin": { + "key": "{{ heketi_admin_key }}" + }, + "_user": "User only has access to /volumes endpoint", + "user": { + "key": "{{ heketi_user_key }}" + } + }, + + "_glusterfs_comment": "GlusterFS Configuration", + "glusterfs": { + "_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh", + "executor": "kubernetes", + + "_db_comment": "Database file name", + "db": "/var/lib/heketi/heketi.db", + + "kubeexec": { + "rebalance_on_expansion": true + }, + + "sshexec": { + "rebalance_on_expansion": true, + "keyfile": "/etc/heketi/private_key", + "fstab": "/etc/fstab", + "port": "22", + "user": "root", + "sudo": false + } + }, + + "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.", + "backup_db_to_kube_secret": false +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 b/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 new file mode 100644 index 00000000000..c2b64cf6942 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gluster + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/glusterfs +parameters: + resturl: "http://{{ endpoint_address }}:8080" + restuser: "admin" + restuserkey: "{{ heketi_admin_key }}" diff --git a/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 new file mode 100644 index 00000000000..b0ac29d7b3b --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 @@ -0,0 +1,34 @@ +{ + "clusters": [ + { + "nodes": [ +{% set nodeblocks = [] %} +{% for node in nodes %} +{% set nodeblock %} + { + "node": { + "hostnames": { + "manage": [ + "{{ node }}" + ], + "storage": [ + "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}" + ] + }, + "zone": 1 + }, + "devices": [ + { + "name": "{{ hostvars[node]['disk_volume_device_1'] }}", + "destroydata": false + } + ] + } +{% endset %} +{% if nodeblocks.append(nodeblock) %}{% endif %} +{% endfor %} +{{ nodeblocks|join(',') }} + ] + } + ] +} diff --git a/contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml b/contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml new file mode 100644 index 00000000000..01e03660c4d --- /dev/null +++ b/contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: "Install lvm utils (RedHat)" + become: true + yum: + name: "lvm2" + state: "present" + when: "ansible_os_family == 'RedHat'" + +- name: "Install lvm utils (Debian)" + become: true + apt: + name: "lvm2" + state: "present" + when: "ansible_os_family == 'Debian'" + +- name: "Get volume group information." + become: true + shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2" + register: "volume_groups" + ignore_errors: true + changed_when: false + +- name: "Remove volume groups." + become: true + command: "vgremove {{ volume_group }} --yes" + with_items: "{{ volume_groups.stdout_lines }}" + loop_control: { loop_var: "volume_group" } + +- name: "Remove physical volume from cluster disks." + become: true + command: "pvremove {{ disk_volume_device_1 }} --yes" + ignore_errors: true + +- name: "Remove lvm utils (RedHat)" + become: true + yum: + name: "lvm2" + state: "absent" + when: "ansible_os_family == 'RedHat'" + +- name: "Remove lvm utils (Debian)" + become: true + apt: + name: "lvm2" + state: "absent" + when: "ansible_os_family == 'Debian'" diff --git a/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml new file mode 100644 index 00000000000..ddc56b256ad --- /dev/null +++ b/contrib/network-storage/heketi/roles/tear-down/tasks/main.yml @@ -0,0 +1,51 @@ +--- +- name: "Remove storage class." + command: "{{ bin_dir }}/kubectl delete storageclass gluster" + ignore_errors: true +- name: "Tear down heketi." + command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\"" + ignore_errors: true +- name: "Tear down heketi." + command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\"" + ignore_errors: true +- name: "Tear down bootstrap." + include_tasks: "../provision/tasks/bootstrap/tear-down.yml" +- name: "Ensure there is nothing left over." + command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json" + register: "heketi_result" + until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" + retries: 60 + delay: 5 +- name: "Ensure there is nothing left over." + command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json" + register: "heketi_result" + until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" + retries: 60 + delay: 5 +- name: "Tear down glusterfs." + command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs" + ignore_errors: true +- name: "Remove heketi storage service." + command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints" + ignore_errors: true +- name: "Remove heketi gluster role binding" + command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin" + ignore_errors: true +- name: "Remove heketi config secret" + command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret" + ignore_errors: true +- name: "Remove heketi db backup" + command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup" + ignore_errors: true +- name: "Remove heketi service account" + command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account" + ignore_errors: true +- name: "Get secrets" + command: "{{ bin_dir }}/kubectl get secrets --output=\"json\"" + register: "secrets" + changed_when: false +- name: "Remove heketi storage secret" + vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" } + command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}" + when: "storage_query is defined" + ignore_errors: true diff --git a/contrib/terraform/aws/README.md b/contrib/terraform/aws/README.md index d69811335f1..709d0633faf 100644 --- a/contrib/terraform/aws/README.md +++ b/contrib/terraform/aws/README.md @@ -17,21 +17,20 @@ This project will create: - Export the variables for your AWS credentials or edit `credentials.tfvars`: ``` -export AWS_ACCESS_KEY_ID="www" -export AWS_SECRET_ACCESS_KEY ="xxx" -export AWS_SSH_KEY_NAME="yyy" -export AWS_DEFAULT_REGION="zzz" +export TF_VAR_AWS_ACCESS_KEY_ID="www" +export TF_VAR_AWS_SECRET_ACCESS_KEY ="xxx" +export TF_VAR_AWS_SSH_KEY_NAME="yyy" +export TF_VAR_AWS_DEFAULT_REGION="zzz" ``` - Rename `contrib/terraform/aws/terraform.tfvars.example` to `terraform.tfvars` - Update `contrib/terraform/aws/terraform.tfvars` with your data. By default, the Terraform scripts use CoreOS as base image. If you want to change this behaviour, see note "Using other distrib than CoreOs" below. -- Allocate a new AWS Elastic IP. Use this for your `loadbalancer_apiserver_address` value (below) - Create an AWS EC2 SSH Key - Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials Example: ```commandline -terraform apply -var-file=credentials.tfvars -var 'loadbalancer_apiserver_address=34.212.228.77' +terraform apply -var-file=credentials.tfvars ``` - Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory` @@ -46,7 +45,7 @@ ssh -F ./ssh-bastion.conf user@$ip Example (this one assumes you are using CoreOS) ```commandline -ansible-playbook -i ./inventory/hosts ./cluster.yml -e ansible_ssh_user=core -e bootstrap_os=coreos -b --become-user=root --flush-cache +ansible-playbook -i ./inventory/hosts ./cluster.yml -e ansible_user=core -e bootstrap_os=coreos -b --become-user=root --flush-cache ``` ***Using other distrib than CoreOs*** If you want to use another distribution than CoreOS, you can modify the search filters of the 'data "aws_ami" "distro"' in variables.tf. diff --git a/contrib/terraform/aws/create-infrastructure.tf b/contrib/terraform/aws/create-infrastructure.tf index 9c0617d8485..1ff584f0c37 100644 --- a/contrib/terraform/aws/create-infrastructure.tf +++ b/contrib/terraform/aws/create-infrastructure.tf @@ -181,7 +181,7 @@ data "template_file" "inventory" { resource "null_resource" "inventories" { provisioner "local-exec" { - command = "echo '${data.template_file.inventory.rendered}' > ../../../inventory/hosts" + command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}" } triggers { diff --git a/contrib/terraform/aws/terraform.tfvars b/contrib/terraform/aws/terraform.tfvars index 99ea64eedaf..c5b1dbff1b1 100644 --- a/contrib/terraform/aws/terraform.tfvars +++ b/contrib/terraform/aws/terraform.tfvars @@ -31,3 +31,5 @@ default_tags = { # Env = "devtest" # Product = "kubernetes" } + +inventory_file = "../../../inventory/hosts" diff --git a/contrib/terraform/aws/variables.tf b/contrib/terraform/aws/variables.tf index 58dd3138847..37aab2bae0f 100644 --- a/contrib/terraform/aws/variables.tf +++ b/contrib/terraform/aws/variables.tf @@ -103,3 +103,7 @@ variable "default_tags" { description = "Default tags for all resources" type = "map" } + +variable "inventory_file" { + description = "Where to store the generated inventory file" +} diff --git a/contrib/terraform/group_vars b/contrib/terraform/group_vars index febd29cb3f6..4dd828e8e58 120000 --- a/contrib/terraform/group_vars +++ b/contrib/terraform/group_vars @@ -1 +1 @@ -../../inventory/group_vars \ No newline at end of file +../../inventory/local/group_vars \ No newline at end of file diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index ed11bef1ef7..15b101fe17c 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -32,7 +32,11 @@ floating IP addresses or not. - Kubernetes worker nodes Note that the Ansible script will report an invalid configuration if you wind up -with an even number of etcd instances since that is not a valid configuration. +with an even number of etcd instances since that is not a valid configuration. This +restriction includes standalone etcd nodes that are deployed in a cluster along with +master nodes with etcd replicas. As an example, if you have three master nodes with +etcd replicas and three standalone etcd nodes, the script will fail since there are +now six total etcd replicas. ### GlusterFS The Terraform configuration supports provisioning of an optional GlusterFS @@ -135,7 +139,7 @@ the one you want to use with the environment variable `OS_CLOUD`: export OS_CLOUD=mycloud ``` -##### Openrc method (deprecated) +##### Openrc method When using classic environment variables, Terraform uses default `OS_*` environment variables. A script suitable for your environment may be available @@ -218,6 +222,8 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tf`. |`number_of_bastions` | Number of bastion hosts to create. Scripts assume this is really just zero or one | |`number_of_gfs_nodes_no_floating_ip` | Number of gluster servers to provision. | | `gfs_volume_size_in_gb` | Size of the non-ephemeral volumes to be attached to store the GlusterFS bricks | +|`supplementary_master_groups` | To add ansible groups to the masters, such as `kube-node` for tainting them as nodes, empty by default. | +|`supplementary_node_groups` | To add ansible groups to the nodes, such as `kube-ingress` for running ingress controller pods, empty by default. | #### Terraform state files @@ -299,11 +305,15 @@ If you have deployed and destroyed a previous iteration of your cluster, you wil #### Bastion host -If you are not using a bastion host, but not all of your nodes have floating IPs, create a file `inventory/$CLUSTER/group_vars/no-floating.yml` with the following content. Use one of your nodes with a floating IP (this should have been output at the end of the Terraform step) and the appropriate user for that OS, or if you have another jump host, use that. +Bastion access will be determined by: -``` -ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -W %h:%p -q USER@MASTER_IP"' -``` + - Your choice on the amount of bastion hosts (set by `number_of_bastions` terraform variable). + - The existence of nodes/masters with floating IPs (set by `number_of_k8s_masters`, `number_of_k8s_nodes`, `number_of_k8s_masters_no_etcd` terraform variables). + +If you have a bastion host, your ssh traffic will be directly routed through it. This is regardless of whether you have masters/nodes with a floating IP assigned. +If you don't have a bastion host, but at least one of your masters/nodes have a floating IP, then ssh traffic will be tunneled by one of these machines. + +So, either a bastion host, or at least master/node with a floating IP are required. #### Test access diff --git a/contrib/terraform/openstack/ansible_bastion_template.txt b/contrib/terraform/openstack/ansible_bastion_template.txt index cdf0120668a..a304b2c9d5d 100644 --- a/contrib/terraform/openstack/ansible_bastion_template.txt +++ b/contrib/terraform/openstack/ansible_bastion_template.txt @@ -1 +1 @@ -ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -W %h:%p -q USER@BASTION_ADDRESS"' +ansible_ssh_common_args: "-o ProxyCommand='ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q USER@BASTION_ADDRESS {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}'" diff --git a/contrib/terraform/openstack/kubespray.tf b/contrib/terraform/openstack/kubespray.tf index e0dbfd02de7..8e5d05adfbe 100644 --- a/contrib/terraform/openstack/kubespray.tf +++ b/contrib/terraform/openstack/kubespray.tf @@ -3,6 +3,7 @@ module "network" { external_net = "${var.external_net}" network_name = "${var.network_name}" + subnet_cidr = "${var.subnet_cidr}" cluster_name = "${var.cluster_name}" dns_nameservers = "${var.dns_nameservers}" } @@ -24,6 +25,7 @@ module "compute" { source = "modules/compute" cluster_name = "${var.cluster_name}" + az_list = "${var.az_list}" number_of_k8s_masters = "${var.number_of_k8s_masters}" number_of_k8s_masters_no_etcd = "${var.number_of_k8s_masters_no_etcd}" number_of_etcd = "${var.number_of_etcd}" @@ -48,6 +50,8 @@ module "compute" { k8s_master_fips = "${module.ips.k8s_master_fips}" k8s_node_fips = "${module.ips.k8s_node_fips}" bastion_fips = "${module.ips.bastion_fips}" + supplementary_master_groups = "${var.supplementary_master_groups}" + supplementary_node_groups = "${var.supplementary_node_groups}" network_id = "${module.network.router_id}" } diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index e0a8eab4a6b..05026ed0b2f 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -59,6 +59,17 @@ resource "openstack_compute_secgroup_v2" "k8s" { self = true } } +resource "openstack_compute_secgroup_v2" "worker" { + name = "${var.cluster_name}-k8s-worker" + description = "${var.cluster_name} - Kubernetes worker nodes" + + rule { + ip_protocol = "tcp" + from_port = "30000" + to_port = "32767" + cidr = "0.0.0.0/0" + } +} resource "openstack_compute_instance_v2" "bastion" { name = "${var.cluster_name}-bastion-${count.index+1}" @@ -83,7 +94,7 @@ resource "openstack_compute_instance_v2" "bastion" { } provisioner "local-exec" { - command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > contrib/terraform/openstack/group_vars/no-floating.yml" + command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > contrib/terraform/group_vars/no-floating.yml" } } @@ -91,6 +102,7 @@ resource "openstack_compute_instance_v2" "bastion" { resource "openstack_compute_instance_v2" "k8s_master" { name = "${var.cluster_name}-k8s-master-${count.index+1}" count = "${var.number_of_k8s_masters}" + availability_zone = "${element(var.az_list, count.index)}" image_name = "${var.image}" flavor_id = "${var.flavor_k8s_master}" key_pair = "${openstack_compute_keypair_v2.k8s.name}" @@ -107,15 +119,20 @@ resource "openstack_compute_instance_v2" "k8s_master" { metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "etcd,kube-master,k8s-cluster,vault" + kubespray_groups = "etcd,kube-master,${var.supplementary_master_groups},k8s-cluster,vault" depends_on = "${var.network_id}" } + provisioner "local-exec" { + command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element( concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > contrib/terraform/group_vars/no-floating.yml" + } + } resource "openstack_compute_instance_v2" "k8s_master_no_etcd" { name = "${var.cluster_name}-k8s-master-ne-${count.index+1}" count = "${var.number_of_k8s_masters_no_etcd}" + availability_zone = "${element(var.az_list, count.index)}" image_name = "${var.image}" flavor_id = "${var.flavor_k8s_master}" key_pair = "${openstack_compute_keypair_v2.k8s.name}" @@ -125,20 +142,26 @@ resource "openstack_compute_instance_v2" "k8s_master_no_etcd" { } security_groups = ["${openstack_compute_secgroup_v2.k8s_master.name}", + "${openstack_compute_secgroup_v2.bastion.name}", "${openstack_compute_secgroup_v2.k8s.name}", ] metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "kube-master,k8s-cluster,vault" + kubespray_groups = "kube-master,${var.supplementary_master_groups},k8s-cluster,vault" depends_on = "${var.network_id}" } + provisioner "local-exec" { + command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element( concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > contrib/terraform/group_vars/no-floating.yml" + } + } resource "openstack_compute_instance_v2" "etcd" { name = "${var.cluster_name}-etcd-${count.index+1}" count = "${var.number_of_etcd}" + availability_zone = "${element(var.az_list, count.index)}" image_name = "${var.image}" flavor_id = "${var.flavor_etcd}" key_pair = "${openstack_compute_keypair_v2.k8s.name}" @@ -160,6 +183,7 @@ resource "openstack_compute_instance_v2" "etcd" { resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" { name = "${var.cluster_name}-k8s-master-nf-${count.index+1}" count = "${var.number_of_k8s_masters_no_floating_ip}" + availability_zone = "${element(var.az_list, count.index)}" image_name = "${var.image}" flavor_id = "${var.flavor_k8s_master}" key_pair = "${openstack_compute_keypair_v2.k8s.name}" @@ -175,7 +199,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" { metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "etcd,kube-master,k8s-cluster,vault,no-floating" + kubespray_groups = "etcd,kube-master,${var.supplementary_master_groups},k8s-cluster,vault,no-floating" depends_on = "${var.network_id}" } @@ -184,6 +208,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" { resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" { name = "${var.cluster_name}-k8s-master-ne-nf-${count.index+1}" count = "${var.number_of_k8s_masters_no_floating_ip_no_etcd}" + availability_zone = "${element(var.az_list, count.index)}" image_name = "${var.image}" flavor_id = "${var.flavor_k8s_master}" key_pair = "${openstack_compute_keypair_v2.k8s.name}" @@ -198,7 +223,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" { metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "kube-master,k8s-cluster,vault,no-floating" + kubespray_groups = "kube-master,${var.supplementary_master_groups},k8s-cluster,vault,no-floating" depends_on = "${var.network_id}" } @@ -207,6 +232,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" { resource "openstack_compute_instance_v2" "k8s_node" { name = "${var.cluster_name}-k8s-node-${count.index+1}" count = "${var.number_of_k8s_nodes}" + availability_zone = "${element(var.az_list, count.index)}" image_name = "${var.image}" flavor_id = "${var.flavor_k8s_node}" key_pair = "${openstack_compute_keypair_v2.k8s.name}" @@ -217,20 +243,26 @@ resource "openstack_compute_instance_v2" "k8s_node" { security_groups = ["${openstack_compute_secgroup_v2.k8s.name}", "${openstack_compute_secgroup_v2.bastion.name}", + "${openstack_compute_secgroup_v2.worker.name}", "default", ] metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "kube-node,k8s-cluster" + kubespray_groups = "kube-node,k8s-cluster,${var.supplementary_node_groups}" depends_on = "${var.network_id}" } + provisioner "local-exec" { + command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element( concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > contrib/terraform/group_vars/no-floating.yml" + } + } resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" { name = "${var.cluster_name}-k8s-node-nf-${count.index+1}" count = "${var.number_of_k8s_nodes_no_floating_ip}" + availability_zone = "${element(var.az_list, count.index)}" image_name = "${var.image}" flavor_id = "${var.flavor_k8s_node}" key_pair = "${openstack_compute_keypair_v2.k8s.name}" @@ -240,12 +272,13 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" { } security_groups = ["${openstack_compute_secgroup_v2.k8s.name}", + "${openstack_compute_secgroup_v2.worker.name}", "default", ] metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "kube-node,k8s-cluster,no-floating" + kubespray_groups = "kube-node,k8s-cluster,no-floating,${var.supplementary_node_groups}" depends_on = "${var.network_id}" } @@ -279,6 +312,7 @@ resource "openstack_blockstorage_volume_v2" "glusterfs_volume" { resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" { name = "${var.cluster_name}-gfs-node-nf-${count.index+1}" count = "${var.number_of_gfs_nodes_no_floating_ip}" + availability_zone = "${element(var.az_list, count.index)}" image_name = "${var.image_gfs}" flavor_id = "${var.flavor_gfs_node}" key_pair = "${openstack_compute_keypair_v2.k8s.name}" diff --git a/contrib/terraform/openstack/modules/compute/variables.tf b/contrib/terraform/openstack/modules/compute/variables.tf index 518e1506945..50a6e496cb8 100644 --- a/contrib/terraform/openstack/modules/compute/variables.tf +++ b/contrib/terraform/openstack/modules/compute/variables.tf @@ -1,5 +1,9 @@ variable "cluster_name" {} +variable "az_list" { + type = "list" +} + variable "number_of_k8s_masters" {} variable "number_of_k8s_masters_no_etcd" {} @@ -55,3 +59,11 @@ variable "k8s_node_fips" { variable "bastion_fips" { type = "list" } + +variable "supplementary_master_groups" { + default = "" +} + +variable "supplementary_node_groups" { + default = "" +} diff --git a/contrib/terraform/openstack/modules/network/main.tf b/contrib/terraform/openstack/modules/network/main.tf index 2c461c78483..7c02869d4d7 100644 --- a/contrib/terraform/openstack/modules/network/main.tf +++ b/contrib/terraform/openstack/modules/network/main.tf @@ -12,7 +12,7 @@ resource "openstack_networking_network_v2" "k8s" { resource "openstack_networking_subnet_v2" "k8s" { name = "${var.cluster_name}-internal-network" network_id = "${openstack_networking_network_v2.k8s.id}" - cidr = "10.0.0.0/24" + cidr = "${var.subnet_cidr}" ip_version = 4 dns_nameservers = "${var.dns_nameservers}" } diff --git a/contrib/terraform/openstack/modules/network/outputs.tf b/contrib/terraform/openstack/modules/network/outputs.tf index a426202b982..e56a792c21c 100644 --- a/contrib/terraform/openstack/modules/network/outputs.tf +++ b/contrib/terraform/openstack/modules/network/outputs.tf @@ -2,6 +2,6 @@ output "router_id" { value = "${openstack_networking_router_interface_v2.k8s.id}" } -output "network_id" { +output "subnet_id" { value = "${openstack_networking_subnet_v2.k8s.id}" } diff --git a/contrib/terraform/openstack/modules/network/variables.tf b/contrib/terraform/openstack/modules/network/variables.tf index a7952bced71..6494358aa3f 100644 --- a/contrib/terraform/openstack/modules/network/variables.tf +++ b/contrib/terraform/openstack/modules/network/variables.tf @@ -7,3 +7,5 @@ variable "cluster_name" {} variable "dns_nameservers" { type = "list" } + +variable "subnet_cidr" {} diff --git a/contrib/terraform/openstack/sample-inventory/cluster.tf b/contrib/terraform/openstack/sample-inventory/cluster.tf index 7830d2159fb..a793bfaa598 100644 --- a/contrib/terraform/openstack/sample-inventory/cluster.tf +++ b/contrib/terraform/openstack/sample-inventory/cluster.tf @@ -41,5 +41,6 @@ number_of_k8s_nodes_no_floating_ip = 4 # networking network_name = "" external_net = "" +subnet_cidr = "" floatingip_pool = "" diff --git a/contrib/terraform/openstack/variables.tf b/contrib/terraform/openstack/variables.tf index 925750ab1b3..dc4ddae9056 100644 --- a/contrib/terraform/openstack/variables.tf +++ b/contrib/terraform/openstack/variables.tf @@ -2,6 +2,12 @@ variable "cluster_name" { default = "example" } +variable "az_list" { + description = "List of Availability Zones available in your OpenStack cluster" + type = "list" + default = ["nova"] +} + variable "number_of_bastions" { default = 1 } @@ -97,6 +103,12 @@ variable "network_name" { default = "internal" } +variable "subnet_cidr" { + description = "Subnet CIDR block." + type = "string" + default = "10.0.0.0/24" +} + variable "dns_nameservers" { description = "An array of DNS name server names used by hosts in this subnet." type = "list" @@ -111,3 +123,13 @@ variable "floatingip_pool" { variable "external_net" { description = "uuid of the external/public network" } + +variable "supplementary_master_groups" { + description = "supplementary kubespray ansible groups for masters, such kube-node" + default = "" +} + +variable "supplementary_node_groups" { + description = "supplementary kubespray ansible groups for worker nodes, such as kube-ingress" + default = "" +} diff --git a/contrib/terraform/terraform.py b/contrib/terraform/terraform.py index 955d5155b40..6feaed42afb 100755 --- a/contrib/terraform/terraform.py +++ b/contrib/terraform/terraform.py @@ -706,6 +706,10 @@ def query_list(hosts): for name, attrs, hostgroups in hosts: for group in set(hostgroups): + # Ansible 2.6.2 stopped supporting empty group names: https://github.com/ansible/ansible/pull/42584/commits/d4cd474b42ed23d8f8aabb2a7f84699673852eaf + # Empty group name defaults to "all" in Ansible < 2.6.2 so we alter empty group names to "all" + if not group: group = "all" + groups[group].setdefault('hosts', []) groups[group]['hosts'].append(name) diff --git a/docs/ansible.md b/docs/ansible.md index 5e17147bedc..21583b5ee30 100644 --- a/docs/ansible.md +++ b/docs/ansible.md @@ -123,7 +123,6 @@ The following tags are defined in playbooks: | hyperkube | Manipulations with K8s hyperkube image | k8s-pre-upgrade | Upgrading K8s cluster | k8s-secrets | Configuring K8s certs/keys -| kpm | Installing K8s apps definitions with KPM | kube-apiserver | Configuring static pod kube-apiserver | kube-controller-manager | Configuring static pod kube-controller-manager | kubectl | Installing kubectl and bash completion @@ -159,7 +158,7 @@ And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/reso ``` ansible-playbook -i inventory/sample/hosts.ini -e dnsmasq_dns_server='' cluster.yml --tags resolvconf ``` -And this prepares all container images localy (at the ansible runner node) without installing +And this prepares all container images locally (at the ansible runner node) without installing or upgrading related stuff or trying to upload container to K8s cluster nodes: ``` ansible-playbook -i inventory/sample/hosts.ini cluster.yml \ diff --git a/docs/aws.md b/docs/aws.md index 8564f6f1038..28cdc89deb2 100644 --- a/docs/aws.md +++ b/docs/aws.md @@ -5,7 +5,7 @@ To deploy kubespray on [AWS](https://aws.amazon.com/) uncomment the `cloud_provi Prior to creating your instances, you **must** ensure that you have created IAM roles and policies for both "kubernetes-master" and "kubernetes-node". You can find the IAM policies [here](https://github.com/kubernetes-incubator/kubespray/tree/master/contrib/aws_iam/). See the [IAM Documentation](https://aws.amazon.com/documentation/iam/) if guidance is needed on how to set these up. When you bring your instances online, associate them with the respective IAM role. Nodes that are only to be used for Etcd do not need a role. -You would also need to tag the resources in your VPC accordingly for the aws provider to utilize them. Tag the subnets and all instances that kubernetes will be run on with key `kubernetes.io/cluster/$cluster_name` (`$cluster_name` must be a unique identifier for the cluster). Tag the subnets that must be targetted by external ELBs with the key `kubernetes.io/role/elb` and internal ELBs with the key `kubernetes.io/role/internal-elb`. +You would also need to tag the resources in your VPC accordingly for the aws provider to utilize them. Tag the subnets, route tables and all instances that kubernetes will be run on with key `kubernetes.io/cluster/$cluster_name` (`$cluster_name` must be a unique identifier for the cluster). Tag the subnets that must be targetted by external ELBs with the key `kubernetes.io/role/elb` and internal ELBs with the key `kubernetes.io/role/internal-elb`. Make sure your VPC has both DNS Hostnames support and Private DNS enabled. diff --git a/docs/calico.md b/docs/calico.md index 7992e57eb6c..f6adaa6622f 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -1,6 +1,13 @@ Calico =========== +--- + **N.B. Version 2.6.5 upgrade to 3.1.1 is upgrading etcd store to etcdv3** + If you create automated backups of etcdv2 please switch for creating etcdv3 backups, as kubernetes and calico now uses etcdv3 + After migration you can check `/tmp/calico_upgrade/` directory for converted items to etcdv3. + **PLEASE TEST upgrade before upgrading production cluster.** + --- + Check if the calico-node container is running ``` @@ -86,7 +93,7 @@ To do so you can deploy BGP route reflectors and peer `calico-node` with them as recommended here: * https://hub.docker.com/r/calico/routereflector/ -* http://docs.projectcalico.org/v2.0/reference/private-cloud/l3-interconnect-fabric +* https://docs.projectcalico.org/v3.1/reference/private-cloud/l3-interconnect-fabric You need to edit your inventory and add: @@ -169,3 +176,12 @@ By default the felix agent(calico-node) will abort if the Kernel RPF setting is ``` calico_node_ignorelooserpf: true ``` + +Note that in OpenStack you must allow `ipip` traffic in your security groups, +otherwise you will experience timeouts. +To do this you must add a rule which allows it, for example: + +``` +neutron security-group-rule-create --protocol 4 --direction egress k8s-a0tp4t +neutron security-group-rule-create --protocol 4 --direction igress k8s-a0tp4t +``` diff --git a/docs/cri-o.md b/docs/cri-o.md new file mode 100644 index 00000000000..43391768aa4 --- /dev/null +++ b/docs/cri-o.md @@ -0,0 +1,31 @@ +cri-o +=============== + +cri-o is container developed by kubernetes project. +Currently, only basic function is supported for cri-o. + +* cri-o is supported kubernetes 1.11.1 or later. +* helm and other feature may not be supported due to docker dependency. +* scale.yml and upgrade-cluster.yml are not supported. + +helm and other feature may not be supported due to docker dependency. + +Use cri-o instead of docker, set following variable: + +#### all.yml + +``` +kubeadm_enabled: true +... +download_container: false +skip_downloads: false +``` + +#### k8s-cluster.yml + +``` +etcd_deployment_type: host +kubelet_deployment_type: host +container_manager: crio +``` + diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 6215114af40..92689eee5b7 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -52,16 +52,24 @@ You can modify how Kubespray sets up DNS for your cluster with the variables ``d ## dns_mode ``dns_mode`` configures how Kubespray will setup cluster DNS. There are four modes available: -#### dnsmasq_kubedns (default) +#### dnsmasq_kubedns This installs an additional dnsmasq DaemonSet which gives more flexibility and lifts some limitations (e.g. number of nameservers). Kubelet is instructed to use dnsmasq instead of kubedns/skydns. It is configured to forward all DNS queries belonging to cluster services to kubedns/skydns. All other queries are forwardet to the nameservers found in ``upstream_dns_servers`` or ``default_resolver`` -#### kubedns +#### kubedns (default) This does not install the dnsmasq DaemonSet and instructs kubelet to directly use kubedns/skydns for all queries. +#### coredns +This does not install the dnsmasq DaemonSet and instructs kubelet to directly use CoreDNS for +all queries. + +#### coredns_dual +This does not install the dnsmasq DaemonSet and instructs kubelet to directly use CoreDNS for +all queries. It will also deploy a secondary CoreDNS stack + #### manual This does not install dnsmasq or kubedns, but allows you to specify `manual_dns_server`, which will be configured on nodes for handling Pod DNS. diff --git a/docs/getting-started.md b/docs/getting-started.md index 961d1a9cfd8..98120b3d749 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -18,11 +18,9 @@ certain threshold. Run `python3 contrib/inventory_builder/inventory.py help` hel Example inventory generator usage: -``` -cp -r inventory/sample inventory/mycluster -declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) -CONFIG_FILE=inventory/mycluster/hosts.ini python3 contrib/inventory_builder/inventory.py ${IPS[@]} -``` + cp -r inventory/sample inventory/mycluster + declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) + CONFIG_FILE=inventory/mycluster/hosts.ini python3 contrib/inventory_builder/inventory.py ${IPS[@]} Starting custom deployment -------------------------- @@ -30,45 +28,70 @@ Starting custom deployment Once you have an inventory, you may want to customize deployment data vars and start the deployment: -**IMPORTANT: Edit my_inventory/groups_vars/*.yaml to override data vars** +**IMPORTANT**: Edit my\_inventory/groups\_vars/\*.yaml to override data vars: -``` -ansible-playbook -i inventory/mycluster/hosts.ini cluster.yml -b -v \ - --private-key=~/.ssh/private_key -``` + ansible-playbook -i inventory/mycluster/hosts.ini cluster.yml -b -v \ + --private-key=~/.ssh/private_key See more details in the [ansible guide](ansible.md). Adding nodes ------------ -You may want to add **worker** nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters. +You may want to add worker, master or etcd nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters. + +- Add the new worker node to your inventory in the appropriate group (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/intro_dynamic_inventory.html)). +- Run the ansible-playbook command, substituting `scale.yml` for `cluster.yml`: + + ansible-playbook -i inventory/mycluster/hosts.ini scale.yml -b -v \ + --private-key=~/.ssh/private_key + +Remove nodes +------------ + +You may want to remove **worker** nodes to your existing cluster. This can be done by re-running the `remove-node.yml` playbook. First, all nodes will be drained, then stop some kubernetes services and delete some certificates, and finally execute the kubectl command to delete these nodes. This can be combined with the add node function, This is generally helpful when doing something like autoscaling your clusters. Of course if a node is not working, you can remove the node and install it again. + +Add worker nodes to the list under kube-node if you want to delete them (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/intro_dynamic_inventory.html)). -- Add the new worker node to your inventory under kube-node (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/intro_dynamic_inventory.html)). -- Run the ansible-playbook command, substituting `scale.yml` for `cluster.yml`: + ansible-playbook -i inventory/mycluster/hosts.ini remove-node.yml -b -v \ + --private-key=~/.ssh/private_key + + +We support two ways to select the nodes: + +- Use `--extra-vars "node=,"` to select the node you want to delete. +``` +ansible-playbook -i inventory/mycluster/hosts.ini remove-node.yml -b -v \ + --private-key=~/.ssh/private_key \ + --extra-vars "node=nodename,nodename2" +``` +or +- Use `--limit nodename,nodename2` to select the node ``` -ansible-playbook -i inventory/mycluster/hosts.ini scale.yml -b -v \ - --private-key=~/.ssh/private_key +ansible-playbook -i inventory/mycluster/hosts.ini remove-node.yml -b -v \ + --private-key=~/.ssh/private_key \ + --limit nodename,nodename2" ``` Connecting to Kubernetes ------------------------ + By default, Kubespray configures kube-master hosts with insecure access to kube-apiserver via port 8080. A kubeconfig file is not necessary in this case, -because kubectl will use http://localhost:8080 to connect. The kubeconfig files +because kubectl will use to connect. The kubeconfig files generated will point to localhost (on kube-masters) and kube-node hosts will connect either to a localhost nginx proxy or to a loadbalancer if configured. More details on this process are in the [HA guide](ha-mode.md). -Kubespray permits connecting to the cluster remotely on any IP of any -kube-master host on port 6443 by default. However, this requires -authentication. One could generate a kubeconfig based on one installed +Kubespray permits connecting to the cluster remotely on any IP of any +kube-master host on port 6443 by default. However, this requires +authentication. One could generate a kubeconfig based on one installed kube-master hosts (needs improvement) or connect with a username and password. By default, a user with admin rights is created, named `kube`. -The password can be viewed after deployment by looking at the file -`PATH_TO_KUBESPRAY/credentials/kube_user`. This contains a randomly generated +The password can be viewed after deployment by looking at the file +`{{ credentials_dir }}/kube_user.creds` (`credentials_dir` is set to `{{ inventory_dir }}/credentials` by default). This contains a randomly generated password. If you wish to set your own password, just precreate/modify this -file yourself. +file yourself. For more information on kubeconfig and accessing a Kubernetes cluster, refer to the Kubernetes [documentation](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/). @@ -77,29 +100,33 @@ Accessing Kubernetes Dashboard ------------------------------ As of kubernetes-dashboard v1.7.x: -* New login options that use apiserver auth proxying of token/basic/kubeconfig by default -* Requires RBAC in authorization_modes -* Only serves over https -* No longer available at https://first_master:6443/ui until apiserver is updated with the https proxy URL + +- New login options that use apiserver auth proxying of token/basic/kubeconfig by default +- Requires RBAC in authorization\_modes +- Only serves over https +- No longer available at until apiserver is updated with the https proxy URL If the variable `dashboard_enabled` is set (default is true), then you can access the Kubernetes Dashboard at the following URL, You will be prompted for credentials: -https://first_master:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login + Or you can run 'kubectl proxy' from your local machine to access dashboard in your browser from: -http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login + -It is recommended to access dashboard from behind a gateway (like Ingress Controller) that enforces an authentication token. Details and other access options here: https://github.com/kubernetes/dashboard/wiki/Accessing-Dashboard---1.7.X-and-above +It is recommended to access dashboard from behind a gateway (like Ingress Controller) that enforces an authentication token. Details and other access options here: Accessing Kubernetes API ------------------------ The main client of Kubernetes is `kubectl`. It is installed on each kube-master host and can optionally be configured on your ansible host by setting -`kubeconfig_localhost: true` in the configuration. If enabled, kubectl and -admin.conf will appear in the artifacts/ directory after deployment. You can -see a list of nodes by running the following commands: +`kubectl_localhost: true` and `kubeconfig_localhost: true` in the configuration: + +- If `kubectl_localhost` enabled, `kubectl` will download onto `/usr/local/bin/` and setup with bash completion. A helper script `inventory/mycluster/artifacts/kubectl.sh` also created for setup with below `admin.conf`. +- If `kubeconfig_localhost` enabled `admin.conf` will appear in the `inventory/mycluster/artifacts/` directory after deployment. + +You can see a list of nodes by running the following commands: - cd artifacts/ - ./kubectl --kubeconfig admin.conf get nodes + cd inventory/mycluster/artifacts + ./kubectl.sh get nodes -If desired, copy kubectl to your bin dir and admin.conf to ~/.kube/config. +If desired, copy admin.conf to ~/.kube/config. diff --git a/docs/ha-mode.md b/docs/ha-mode.md index f3bc97e1c6e..61963663306 100644 --- a/docs/ha-mode.md +++ b/docs/ha-mode.md @@ -11,12 +11,32 @@ achieve the same goal. Etcd ---- -The `etcd_access_endpoint` fact provides an access pattern for clients. And the -`etcd_multiaccess` (defaults to `True`) group var controls that behavior. -It makes deployed components to access the etcd cluster members -directly: `http://ip1:2379, http://ip2:2379,...`. This mode assumes the clients -do a loadbalancing and handle HA for connections. +In order to use an external loadbalancing (L4/TCP or L7 w/ SSL Passthrough VIP), the following variables need to be overriden in group_vars +* `etcd_access_addresses` +* `etcd_client_url` +* `etcd_cert_alt_names` +* `etcd_cert_alt_ips` + +### Example of a VIP w/ FQDN +```yaml +etcd_access_addresses: https://etcd.example.com:2379 +etcd_client_url: https://etcd.example.com:2379 +etcd_cert_alt_names: + - "etcd.kube-system.svc.{{ dns_domain }}" + - "etcd.kube-system.svc" + - "etcd.kube-system" + - "etcd" + - "etcd.example.com" # This one needs to be added to the default etcd_cert_alt_names +``` + +### Example of a VIP w/o FQDN (IP only) +```yaml +etcd_access_addresses: https://2.3.7.9:2379 +etcd_client_url: https://2.3.7.9:2379 +etcd_cert_alt_ips: + - "2.3.7.9" +``` Kube-apiserver -------------- diff --git a/docs/img/kubernetes-logo.png b/docs/img/kubernetes-logo.png new file mode 100644 index 00000000000..2838a1829ff Binary files /dev/null and b/docs/img/kubernetes-logo.png differ diff --git a/docs/kubernetes-reliability.md b/docs/kubernetes-reliability.md index 82ec65f2e0f..e338272d3bc 100644 --- a/docs/kubernetes-reliability.md +++ b/docs/kubernetes-reliability.md @@ -33,7 +33,7 @@ Kubelet will try to make `nodeStatusUpdateRetry` post attempts. Currently [kubelet.go](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet.go#L102). Kubelet will try to update the status in -[tryUpdateNodeStatus](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet_node_status.go#L345) +[tryUpdateNodeStatus](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet_node_status.go#L312) function. Kubelet uses `http.Client()` Golang method, but has no specified timeout. Thus there may be some glitches when API Server is overloaded while TCP connection is established. diff --git a/docs/netcheck.md b/docs/netcheck.md index 80679cd7310..46038965f78 100644 --- a/docs/netcheck.md +++ b/docs/netcheck.md @@ -25,13 +25,13 @@ There are related application specifc variables: netchecker_port: 31081 agent_report_interval: 15 netcheck_namespace: default -agent_img: "quay.io/l23network/k8s-netchecker-agent:v1.0" -server_img: "quay.io/l23network/k8s-netchecker-server:v1.0" +agent_img: "mirantis/k8s-netchecker-agent:v1.2.2" +server_img: "mirantis/k8s-netchecker-server:v1.2.2" ``` Note that the application verifies DNS resolve for FQDNs comprising only the combination of the ``netcheck_namespace.dns_domain`` vars, for example the -``netchecker-service.default.cluster.local``. If you want to deploy the application +``netchecker-service.default.svc.cluster.local``. If you want to deploy the application to the non default namespace, make sure as well to adjust the ``searchdomains`` var so the resulting search domain records to contain that namespace, like: diff --git a/docs/openstack.md b/docs/openstack.md index 7a4368e2e29..ca1e89ae188 100644 --- a/docs/openstack.md +++ b/docs/openstack.md @@ -3,7 +3,7 @@ OpenStack To deploy kubespray on [OpenStack](https://www.openstack.org/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'openstack'`. -After that make sure to source in your OpenStack credentials like you would do when using `nova-client` by using `source path/to/your/openstack-rc`. +After that make sure to source in your OpenStack credentials like you would do when using `nova-client` or `neutron-client` by using `source path/to/your/openstack-rc` or `. path/to/your/openstack-rc`. The next step is to make sure the hostnames in your `inventory` file are identical to your instance names in OpenStack. Otherwise [cinder](https://wiki.openstack.org/wiki/Cinder) won't work as expected. @@ -12,35 +12,34 @@ Unless you are using calico you can now run the playbook. **Additional step needed when using calico:** -Calico does not encapsulate all packages with the hosts ip addresses. Instead the packages will be routed with the PODs ip addresses directly. +Calico does not encapsulate all packages with the hosts' ip addresses. Instead the packages will be routed with the PODs ip addresses directly. + OpenStack will filter and drop all packages from ips it does not know to prevent spoofing. -In order to make calico work on OpenStack you will need to tell OpenStack to allow calicos packages by allowing the network it uses. +In order to make calico work on OpenStack you will need to tell OpenStack to allow calico's packages by allowing the network it uses. First you will need the ids of your OpenStack instances that will run kubernetes: - nova list --tenant Your-Tenant + openstack server list --project YOUR_PROJECT +--------------------------------------+--------+----------------------------------+--------+-------------+ | ID | Name | Tenant ID | Status | Power State | +--------------------------------------+--------+----------------------------------+--------+-------------+ | e1f48aad-df96-4bce-bf61-62ae12bf3f95 | k8s-1 | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running | | 725cd548-6ea3-426b-baaa-e7306d3c8052 | k8s-2 | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running | -Then you can use the instance ids to find the connected [neutron](https://wiki.openstack.org/wiki/Neutron) ports: +Then you can use the instance ids to find the connected [neutron](https://wiki.openstack.org/wiki/Neutron) ports (though they are now configured through using OpenStack): - neutron port-list -c id -c device_id + openstack port list -c id -c device_id --project YOUR_PROJECT +--------------------------------------+--------------------------------------+ | id | device_id | +--------------------------------------+--------------------------------------+ | 5662a4e0-e646-47f0-bf88-d80fbd2d99ef | e1f48aad-df96-4bce-bf61-62ae12bf3f95 | | e5ae2045-a1e1-4e99-9aac-4353889449a7 | 725cd548-6ea3-426b-baaa-e7306d3c8052 | -Given the port ids on the left, you can set the `allowed_address_pairs` in neutron. -Note that you have to allow both of `kube_service_addresses` (default `10.233.0.0/18`) -and `kube_pods_subnet` (default `10.233.64.0/18`.) +Given the port ids on the left, you can set the two `allowed_address`(es) in OpenStack. Note that you have to allow both `kube_service_addresses` (default `10.233.0.0/18`) and `kube_pods_subnet` (default `10.233.64.0/18`.) # allow kube_service_addresses and kube_pods_subnet network - neutron port-update 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed_address_pairs list=true type=dict ip_address=10.233.0.0/18 ip_address=10.233.64.0/18 - neutron port-update e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed_address_pairs list=true type=dict ip_address=10.233.0.0/18 ip_address=10.233.64.0/18 + openstack port set 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed_address ip_address=10.233.0.0/18,ip_address=10.233.64.0/18 + openstack port set e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed_address ip_address=10.233.0.0/18,ip_address=10.233.64.0/18 Now you can finally run the playbook. diff --git a/docs/opensuse.md b/docs/opensuse.md new file mode 100644 index 00000000000..88fac3790e3 --- /dev/null +++ b/docs/opensuse.md @@ -0,0 +1,19 @@ +openSUSE Leap 42.3 and Tumbleweed +=============== + +openSUSE Leap installation Notes: + +- Install Ansible + + ``` + sudo zypper ref + sudo zypper -n install ansible + + ``` + +- Install Jinja2 and Python-Netaddr + + ```sudo zypper -n install python-Jinja2 python-netaddr``` + + +Now you can continue with [Preparing your deployment](getting-started.md#starting-custom-deployment) diff --git a/docs/roadmap.md b/docs/roadmap.md index cf8fa2d9a32..a0c1a3ffc62 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -9,7 +9,7 @@ Kubespray's roadmap ### Self deployment (pull-mode) [#320](https://github.com/kubespray/kubespray/issues/320) - the playbook would install and configure docker/rkt and the etcd cluster - the following data would be inserted into etcd: certs,tokens,users,inventory,group_vars. -- a "kubespray" container would be deployed (kubespray-cli, ansible-playbook, kpm) +- a "kubespray" container would be deployed (kubespray-cli, ansible-playbook) - to be discussed, a way to provide the inventory - **self deployment** of the node from inside a container [#321](https://github.com/kubespray/kubespray/issues/321) diff --git a/docs/upgrades.md b/docs/upgrades.md index 6297976ddb5..620e07a6b33 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -81,3 +81,61 @@ kubernetes-apps/rotate_tokens role, only pods in kube-system are destroyed and recreated. All other invalidated service account tokens are cleaned up automatically, but other pods are not deleted out of an abundance of caution for impact to user deployed pods. + +### Component-based upgrades + +A deployer may want to upgrade specific components in order to minimize risk +or save time. This strategy is not covered by CI as of this writing, so it is +not guaranteed to work. + +These commands are useful only for upgrading fully-deployed, healthy, existing +hosts. This will definitely not work for undeployed or partially deployed +hosts. + +Upgrade docker: + +``` +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=docker +``` + +Upgrade etcd: + +``` +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd +``` + +Upgrade vault: + +``` +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=vault +``` + +Upgrade kubelet: + +``` +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=node --skip-tags=k8s-gen-certs,k8s-gen-tokens +``` + +Upgrade Kubernetes master components: + +``` +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=master +``` + +Upgrade network plugins: + +``` +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=network +``` + +Upgrade all add-ons: + +``` +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=apps +``` + +Upgrade just helm (assuming `helm_enabled` is true): + +``` +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=helm +``` diff --git a/docs/vagrant.md b/docs/vagrant.md index 042e8137bd2..de47159fa73 100644 --- a/docs/vagrant.md +++ b/docs/vagrant.md @@ -1,7 +1,7 @@ Vagrant Install ================= -Assuming you have Vagrant (1.9+) installed with virtualbox (it may work +Assuming you have Vagrant (2.0+) installed with virtualbox (it may work with vmware, but is untested) you should be able to launch a 3 node Kubernetes cluster by simply running `$ vagrant up`.
diff --git a/docs/vars.md b/docs/vars.md index 3303f6bcbfe..41248b036c8 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -63,7 +63,8 @@ following default cluster paramters: bits in kube_pods_subnet dictates how many kube-nodes can be in cluster. * *dns_setup* - Enables dnsmasq * *dnsmasq_dns_server* - Cluster IP for dnsmasq (default is 10.233.0.2) -* *skydns_server* - Cluster IP for KubeDNS (default is 10.233.0.3) +* *skydns_server* - Cluster IP for DNS (default is 10.233.0.3) +* *skydns_server_secondary* - Secondary Cluster IP for CoreDNS used with coredns_dual deployment (default is 10.233.0.4) * *cloud_provider* - Enable extra Kubelet option if operating inside GCE or OpenStack (default is unset) * *kube_hostpath_dynamic_provisioner* - Required for use of PetSets type in @@ -105,9 +106,9 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st * *http_proxy/https_proxy/no_proxy* - Proxy variables for deploying behind a proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames that correspond to each node. -* *kubelet_deployment_type* - Controls which platform to deploy kubelet on. +* *kubelet_deployment_type* - Controls which platform to deploy kubelet on. Available options are ``host``, ``rkt``, and ``docker``. ``docker`` mode - is unlikely to work on newer releases. Starting with Kubernetes v1.7 + is unlikely to work on newer releases. Starting with Kubernetes v1.7 series, this now defaults to ``host``. Before v1.7, the default was Docker. This is because of cgroup [issues](https://github.com/kubernetes/kubernetes/issues/43704). * *kubelet_load_modules* - For some things, kubelet needs to load kernel modules. For example, @@ -117,6 +118,14 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st * *kubelet_cgroup_driver* - Allows manual override of the cgroup-driver option for Kubelet. By default autodetection is used to match Docker configuration. +* *node_labels* - Labels applied to nodes via kubelet --node-labels parameter. + For example, labels can be set in the inventory as variables or more widely in group_vars. + *node_labels* must be defined as a dict: +``` +node_labels: + label1_name: label1_value + label2_name: label2_value +``` ##### Custom flags for Kube Components For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example: @@ -136,6 +145,6 @@ The possible vars are: By default, a user with admin rights is created, named `kube`. The password can be viewed after deployment by looking at the file -`PATH_TO_KUBESPRAY/credentials/kube_user`. This contains a randomly generated +`{{ credentials_dir }}/kube_user.creds` (`credentials_dir` is set to `{{ inventory_dir }}/credentials` by default). This contains a randomly generated password. If you wish to set your own password, just precreate/modify this file yourself or change `kube_api_pwd` var. diff --git a/extra_playbooks/build-cephfs-provisioner.yml b/extra_playbooks/build-cephfs-provisioner.yml index 6a72a076e91..a669805c7fd 100644 --- a/extra_playbooks/build-cephfs-provisioner.yml +++ b/extra_playbooks/build-cephfs-provisioner.yml @@ -8,8 +8,8 @@ version: "{{ item.version }}" state: "{{ item.state }}" with_items: - - { state: "present", name: "docker", version: "2.7.0" } - - { state: "present", name: "docker-compose", version: "1.18.0" } + - { state: "present", name: "docker", version: "3.4.1" } + - { state: "present", name: "docker-compose", version: "1.21.2" } - name: CephFS Provisioner | Check Go version shell: | @@ -35,19 +35,19 @@ - name: CephFS Provisioner | Clone repo git: repo: https://github.com/kubernetes-incubator/external-storage.git - dest: "~/go/src/github.com/kubernetes-incubator" - version: 92295a30 - clone: no + dest: "~/go/src/github.com/kubernetes-incubator/external-storage" + version: 06fddbe2 + clone: yes update: yes - name: CephFS Provisioner | Build image shell: | cd ~/go/src/github.com/kubernetes-incubator/external-storage - REGISTRY=quay.io/kubespray/ VERSION=92295a30 make ceph/cephfs + REGISTRY=quay.io/kubespray/ VERSION=06fddbe2 make ceph/cephfs - name: CephFS Provisioner | Push image docker_image: - name: quay.io/kubespray/cephfs-provisioner:92295a30 + name: quay.io/kubespray/cephfs-provisioner:06fddbe2 push: yes retries: 10 diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml deleted file mode 100644 index c107b049f8d..00000000000 --- a/inventory/sample/group_vars/all.yml +++ /dev/null @@ -1,136 +0,0 @@ -# Valid bootstrap options (required): ubuntu, coreos, centos, none -bootstrap_os: none - -#Directory where etcd data stored -etcd_data_dir: /var/lib/etcd - -# Directory where the binaries will be installed -bin_dir: /usr/local/bin - -## The access_ip variable is used to define how other nodes should access -## the node. This is used in flannel to allow other flannel nodes to see -## this node for example. The access_ip is really useful AWS and Google -## environments where the nodes are accessed remotely by the "public" ip, -## but don't know about that address themselves. -#access_ip: 1.1.1.1 - -### LOADBALANCING AND ACCESS MODES -## Enable multiaccess to configure etcd clients to access all of the etcd members directly -## as the "http://hostX:port, http://hostY:port, ..." and ignore the proxy loadbalancers. -## This may be the case if clients support and loadbalance multiple etcd servers natively. -#etcd_multiaccess: true - -### ETCD: disable peer client cert authentication. -# This affects ETCD_PEER_CLIENT_CERT_AUTH variable -#etcd_peer_client_auth: true - -## External LB example config -## apiserver_loadbalancer_domain_name: "elb.some.domain" -#loadbalancer_apiserver: -# address: 1.2.3.4 -# port: 1234 - -## Internal loadbalancers for apiservers -#loadbalancer_apiserver_localhost: true - -## Local loadbalancer should use this port instead, if defined. -## Defaults to kube_apiserver_port (6443) -#nginx_kube_apiserver_port: 8443 - -### OTHER OPTIONAL VARIABLES -## For some things, kubelet needs to load kernel modules. For example, dynamic kernel services are needed -## for mounting persistent volumes into containers. These may not be loaded by preinstall kubernetes -## processes. For example, ceph and rbd backed volumes. Set to true to allow kubelet to load kernel -## modules. -#kubelet_load_modules: false - -## Internal network total size. This is the prefix of the -## entire network. Must be unused in your environment. -#kube_network_prefix: 18 - -## With calico it is possible to distributed routes with border routers of the datacenter. -## Warning : enabling router peering will disable calico's default behavior ('node mesh'). -## The subnets of each nodes will be distributed by the datacenter router -#peer_with_router: false - -## Upstream dns servers used by dnsmasq -#upstream_dns_servers: -# - 8.8.8.8 -# - 8.8.4.4 - -## There are some changes specific to the cloud providers -## for instance we need to encapsulate packets with some network plugins -## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', or 'external' -## When openstack is used make sure to source in the openstack credentials -## like you would do when using nova-client before starting the playbook. -#cloud_provider: - -## When azure is used, you need to also set the following variables. -## see docs/azure.md for details on how to get these values -#azure_tenant_id: -#azure_subscription_id: -#azure_aad_client_id: -#azure_aad_client_secret: -#azure_resource_group: -#azure_location: -#azure_subnet_name: -#azure_security_group_name: -#azure_vnet_name: -#azure_route_table_name: - -## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) -#openstack_blockstorage_version: "v1/v2/auto (default)" -## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. -#openstack_lbaas_enabled: True -#openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP" -## To enable automatic floating ip provisioning, specify a subnet. -#openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default" -## Override default LBaaS behavior -#openstack_lbaas_use_octavia: False -#openstack_lbaas_method: "ROUND_ROBIN" -#openstack_lbaas_provider: "haproxy" -#openstack_lbaas_create_monitor: "yes" -#openstack_lbaas_monitor_delay: "1m" -#openstack_lbaas_monitor_timeout: "30s" -#openstack_lbaas_monitor_max_retries: "3" - -## Uncomment to enable experimental kubeadm deployment mode -#kubeadm_enabled: false -## Set these proxy values in order to update package manager and docker daemon to use proxies -#http_proxy: "" -#https_proxy: "" -## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy -#no_proxy: "" - -## Uncomment this if you want to force overlay/overlay2 as docker storage driver -## Please note that overlay2 is only supported on newer kernels -#docker_storage_options: -s overlay2 - -# Uncomment this if you have more than 3 nameservers, then we'll only use the first 3. -#docker_dns_servers_strict: false - -## Default packages to install within the cluster, f.e: -#kpm_packages: -# - name: kube-system/grafana - -## Certificate Management -## This setting determines whether certs are generated via scripts or whether a -## cluster of Hashicorp's Vault is started to issue certificates (using etcd -## as a backend). Options are "script" or "vault" -#cert_management: script - -# Set to true to allow pre-checks to fail and continue deployment -#ignore_assert_errors: false - -## Etcd auto compaction retention for mvcc key value store in hour -#etcd_compaction_retention: 0 - -## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics. -#etcd_metrics: basic - -## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing. -## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM. -#etcd_memory_limit: "512M" - -# The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable. -#kube_read_only_port: 10255 diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml new file mode 100644 index 00000000000..faf65eb1ad4 --- /dev/null +++ b/inventory/sample/group_vars/all/all.yml @@ -0,0 +1,82 @@ +## Valid bootstrap options (required): ubuntu, coreos, centos, none +## If the OS is not listed here, it means it doesn't require extra/bootstrap steps. +## In example, python is not available on 'coreos' so it must be installed before +## anything else. In the opposite, Debian has already all its dependencies fullfiled, then bootstrap_os should be set to `none`. +bootstrap_os: none + +## Directory where etcd data stored +etcd_data_dir: /var/lib/etcd + +## Directory where the binaries will be installed +bin_dir: /usr/local/bin + +## The access_ip variable is used to define how other nodes should access +## the node. This is used in flannel to allow other flannel nodes to see +## this node for example. The access_ip is really useful AWS and Google +## environments where the nodes are accessed remotely by the "public" ip, +## but don't know about that address themselves. +#access_ip: 1.1.1.1 + + +## External LB example config +## apiserver_loadbalancer_domain_name: "elb.some.domain" +#loadbalancer_apiserver: +# address: 1.2.3.4 +# port: 1234 + +## Internal loadbalancers for apiservers +#loadbalancer_apiserver_localhost: true + +## Local loadbalancer should use this port instead, if defined. +## Defaults to kube_apiserver_port (6443) +#nginx_kube_apiserver_port: 8443 + +### OTHER OPTIONAL VARIABLES +## For some things, kubelet needs to load kernel modules. For example, dynamic kernel services are needed +## for mounting persistent volumes into containers. These may not be loaded by preinstall kubernetes +## processes. For example, ceph and rbd backed volumes. Set to true to allow kubelet to load kernel +## modules. +#kubelet_load_modules: false + +## With calico it is possible to distributed routes with border routers of the datacenter. +## Warning : enabling router peering will disable calico's default behavior ('node mesh'). +## The subnets of each nodes will be distributed by the datacenter router +#peer_with_router: false + +## Upstream dns servers used by dnsmasq +#upstream_dns_servers: +# - 8.8.8.8 +# - 8.8.4.4 + +## There are some changes specific to the cloud providers +## for instance we need to encapsulate packets with some network plugins +## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external' +## When openstack is used make sure to source in the openstack credentials +## like you would do when using nova-client before starting the playbook. +#cloud_provider: + + +## Uncomment to enable experimental kubeadm deployment mode +#kubeadm_enabled: false + +## Set these proxy values in order to update package manager and docker daemon to use proxies +#http_proxy: "" +#https_proxy: "" + +## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy +#no_proxy: "" + +## Certificate Management +## This setting determines whether certs are generated via scripts or whether a +## cluster of Hashicorp's Vault is started to issue certificates (using etcd +## as a backend). Options are "script" or "vault" +#cert_management: script + +## Set to true to allow pre-checks to fail and continue deployment +#ignore_assert_errors: false + +## The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable. +#kube_read_only_port: 10255 + +## Set true to download and cache container +#download_container: true diff --git a/inventory/sample/group_vars/all/azure.yml b/inventory/sample/group_vars/all/azure.yml new file mode 100644 index 00000000000..78d49c9b4d4 --- /dev/null +++ b/inventory/sample/group_vars/all/azure.yml @@ -0,0 +1,14 @@ +## When azure is used, you need to also set the following variables. +## see docs/azure.md for details on how to get these values + +#azure_tenant_id: +#azure_subscription_id: +#azure_aad_client_id: +#azure_aad_client_secret: +#azure_resource_group: +#azure_location: +#azure_subnet_name: +#azure_security_group_name: +#azure_vnet_name: +#azure_vnet_resource_group: +#azure_route_table_name: diff --git a/inventory/sample/group_vars/all/coreos.yml b/inventory/sample/group_vars/all/coreos.yml new file mode 100644 index 00000000000..a48f24ebbc3 --- /dev/null +++ b/inventory/sample/group_vars/all/coreos.yml @@ -0,0 +1,2 @@ +## Does coreos need auto upgrade, default is true +#coreos_auto_upgrade: true diff --git a/inventory/sample/group_vars/all/docker.yml b/inventory/sample/group_vars/all/docker.yml new file mode 100644 index 00000000000..c1a1dd85a6b --- /dev/null +++ b/inventory/sample/group_vars/all/docker.yml @@ -0,0 +1,61 @@ +## Uncomment this if you want to force overlay/overlay2 as docker storage driver +## Please note that overlay2 is only supported on newer kernels +#docker_storage_options: -s overlay2 + +## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7. +docker_container_storage_setup: false + +## It must be define a disk path for docker_container_storage_setup_devs. +## Otherwise docker-storage-setup will be executed incorrectly. +#docker_container_storage_setup_devs: /dev/vdb + +## Uncomment this if you have more than 3 nameservers, then we'll only use the first 3. +docker_dns_servers_strict: false + +# Path used to store Docker data +docker_daemon_graph: "/var/lib/docker" + +## Used to set docker daemon iptables options to true +docker_iptables_enabled: "false" + +# Docker log options +# Rotate container stderr/stdout logs at 50m and keep last 5 +docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" + +# define docker bin_dir +docker_bin_dir: "/usr/bin" + +## An obvious use case is allowing insecure-registry access to self hosted registries. +## Can be ipddress and domain_name. +## example define 172.19.16.11 or mirror.registry.io +#docker_insecure_registries: +# - mirror.registry.io +# - 172.19.16.11 + +## Add other registry,example China registry mirror. +#docker_registry_mirrors: +# - https://registry.docker-cn.com +# - https://mirror.aliyuncs.com + +## If non-empty will override default system MounFlags value. +## This option takes a mount propagation flag: shared, slave +## or private, which control whether mounts in the file system +## namespace set up for docker will receive or propagate mounts +## and unmounts. Leave empty for system default +#docker_mount_flags: + +## A string of extra options to pass to the docker daemon. +## This string should be exactly as you wish it to appear. +docker_options: >- + {%- if docker_insecure_registries is defined -%} + {{ docker_insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} + {%- endif %} + {% if docker_registry_mirrors is defined -%} + {{ docker_registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }} + {%- endif %} + --graph={{ docker_daemon_graph }} {{ docker_log_opts }} + {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %} + --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current + --default-runtime=docker-runc --exec-opt native.cgroupdriver=systemd + --userland-proxy-path=/usr/libexec/docker/docker-proxy-current --signature-verification=false + {%- endif -%} diff --git a/inventory/sample/group_vars/all/oci.yml b/inventory/sample/group_vars/all/oci.yml new file mode 100644 index 00000000000..fd83080dd80 --- /dev/null +++ b/inventory/sample/group_vars/all/oci.yml @@ -0,0 +1,15 @@ +## When Oracle Cloud Infrastructure is used, set these variables +#oci_private_key: +#oci_region_id: +#oci_tenancy_id: +#oci_user_id: +#oci_user_fingerprint: +#oci_compartment_id: +#oci_vnc_id: +#oci_subnet1_id: +#oci_subnet2_id: +## Overide these default behaviors if you wish +#oci_security_list_management: All +# If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint +#oci_use_instance_principals: false +#oci_cloud_controller_version: 0.5.0 diff --git a/inventory/sample/group_vars/all/openstack.yml b/inventory/sample/group_vars/all/openstack.yml new file mode 100644 index 00000000000..6347d0522f2 --- /dev/null +++ b/inventory/sample/group_vars/all/openstack.yml @@ -0,0 +1,15 @@ +## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) +#openstack_blockstorage_version: "v1/v2/auto (default)" +## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. +#openstack_lbaas_enabled: True +#openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP" +## To enable automatic floating ip provisioning, specify a subnet. +#openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default" +## Override default LBaaS behavior +#openstack_lbaas_use_octavia: False +#openstack_lbaas_method: "ROUND_ROBIN" +#openstack_lbaas_provider: "haproxy" +#openstack_lbaas_create_monitor: "yes" +#openstack_lbaas_monitor_delay: "1m" +#openstack_lbaas_monitor_timeout: "30s" +#openstack_lbaas_monitor_max_retries: "3" diff --git a/inventory/sample/group_vars/etcd.yml b/inventory/sample/group_vars/etcd.yml new file mode 100644 index 00000000000..6f5347cb956 --- /dev/null +++ b/inventory/sample/group_vars/etcd.yml @@ -0,0 +1,18 @@ +## Etcd auto compaction retention for mvcc key value store in hour +#etcd_compaction_retention: 0 + +## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics. +#etcd_metrics: basic + +## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing. +## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM. +#etcd_memory_limit: "512M" + +## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than +## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check +## etcd documentation for more information. +#etcd_quota_backend_bytes: "2G" + +### ETCD: disable peer client cert authentication. +# This affects ETCD_PEER_CLIENT_CERT_AUTH variable +#etcd_peer_client_auth: true diff --git a/inventory/sample/group_vars/k8s-cluster/addons.yml b/inventory/sample/group_vars/k8s-cluster/addons.yml new file mode 100644 index 00000000000..7c9057e715e --- /dev/null +++ b/inventory/sample/group_vars/k8s-cluster/addons.yml @@ -0,0 +1,54 @@ +# Kubernetes dashboard +# RBAC required. see docs/getting-started.md for access details. +dashboard_enabled: true + +# Monitoring apps for k8s +efk_enabled: false + +# Helm deployment +helm_enabled: false + +# Registry deployment +registry_enabled: false +# registry_namespace: "{{ system_namespace }}" +# registry_storage_class: "" +# registry_disk_size: "10Gi" + +# Local volume provisioner deployment +local_volume_provisioner_enabled: false +# local_volume_provisioner_namespace: "{{ system_namespace }}" +# local_volume_provisioner_base_dir: /mnt/disks +# local_volume_provisioner_mount_dir: /mnt/disks +# local_volume_provisioner_storage_class: local-storage + +# CephFS provisioner deployment +cephfs_provisioner_enabled: false +# cephfs_provisioner_namespace: "cephfs-provisioner" +# cephfs_provisioner_cluster: ceph +# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789" +# cephfs_provisioner_admin_id: admin +# cephfs_provisioner_secret: secret +# cephfs_provisioner_storage_class: cephfs +# cephfs_provisioner_reclaim_policy: Delete +# cephfs_provisioner_claim_root: /volumes +# cephfs_provisioner_deterministic_names: true + +# Nginx ingress controller deployment +ingress_nginx_enabled: false +# ingress_nginx_host_network: false +# ingress_nginx_nodeselector: +# node-role.kubernetes.io/master: "true" +# ingress_nginx_namespace: "ingress-nginx" +# ingress_nginx_insecure_port: 80 +# ingress_nginx_secure_port: 443 +# ingress_nginx_configmap: +# map-hash-bucket-size: "128" +# ssl-protocols: "SSLv2" +# ingress_nginx_configmap_tcp_services: +# 9000: "default/example-go:8080" +# ingress_nginx_configmap_udp_services: +# 53: "kube-system/kube-dns:53" + +# Cert manager deployment +cert_manager_enabled: false +# cert_manager_namespace: "cert-manager" diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml similarity index 61% rename from inventory/sample/group_vars/k8s-cluster.yml rename to inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index a31963f160c..efc6845b236 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -1,12 +1,11 @@ # Kubernetes configuration dirs and system namespace. # Those are where all the additional config stuff goes -# the kubernetes normally puts in /srv/kubernets. +# the kubernetes normally puts in /srv/kubernetes. # This puts them in a sane location and namespace. -# Editting those values will almost surely break something. +# Editing those values will almost surely break something. kube_config_dir: /etc/kubernetes kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" kube_manifest_dir: "{{ kube_config_dir }}/manifests" -system_namespace: kube-system # This is where all the cert scripts and certs will be located kube_cert_dir: "{{ kube_config_dir }}/ssl" @@ -20,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.9.3 +kube_version: v1.11.3 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) @@ -29,15 +28,18 @@ local_release_dir: "/tmp/releases" retry_stagger: 5 # This is the group that the cert creation scripts chgrp the -# cert files to. Not really changable... +# cert files to. Not really changeable... kube_cert_group: kube-cert # Cluster Loglevel configuration kube_log_level: 2 +# Directory where credentials will be stored +credentials_dir: "{{ inventory_dir }}/credentials" + # Users to create for basic auth in Kubernetes API via HTTP # Optionally add groups for user -kube_api_pwd: "{{ lookup('password', inventory_dir + '/credentials/kube_user length=15 chars=ascii_letters,digits') }}" +kube_api_pwd: "{{ lookup('password', credentials_dir + '/kube_user.creds length=15 chars=ascii_letters,digits') }}" kube_users: kube: pass: "{{kube_api_pwd}}" @@ -59,35 +61,30 @@ kube_users: ## Optional settings for OIDC # kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem # kube_oidc_username_claim: sub +# kube_oidc_username_prefix: oidc: # kube_oidc_groups_claim: groups +# kube_oidc_groups_prefix: oidc: # Choose network plugin (cilium, calico, contiv, weave or flannel) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico -# weave's network password for encryption -# if null then no network encryption -# you can use --extra-vars to pass the password in command line -weave_password: EnterPasswordHere - -# Weave uses consensus mode by default -# Enabling seed mode allow to dynamically add or remove hosts -# https://www.weave.works/docs/net/latest/ipam/ -weave_mode_seed: false - -# This two variable are automatically changed by the weave's role, do not manually change these values -# To reset values : -# weave_seed: uninitialized -# weave_peers: uninitialized -weave_seed: uninitialized -weave_peers: uninitialized - -# Set the MTU of Weave (default 1376, Jumbo Frames: 8916) -weave_mtu: 1376 - -# Enable kubernetes network policies -enable_network_policy: false +# Weave deployment +# weave_password: ~ +# weave_checkpoint_disable: false +# weave_conn_limit: 100 +# weave_hairpin_mode: true +# weave_ipalloc_range: {{ kube_pods_subnet }} +# weave_expect_npc: {{ enable_network_policy }} +# weave_kube_peers: ~ +# weave_ipalloc_init: ~ +# weave_expose_ip: ~ +# weave_metrics_addr: ~ +# weave_status_addr: ~ +# weave_mtu: 1376 +# weave_no_masq_local: true +# weave_extra_args: ~ # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 @@ -111,14 +108,22 @@ kube_apiserver_insecure_port: 8080 # (http) # Kube-proxy proxyMode configuration. # Can be ipvs, iptables -kube_proxy_mode: iptables +kube_proxy_mode: iptables + +# Kube-proxy nodeport address. +# cidr to bind nodeport services. Flag --nodeport-addresses on kube-proxy manifest +kube_proxy_nodeport_addresses: false +# kube_proxy_nodeport_addresses_cidr: 10.0.1.0/24 + +## Encrypting Secret Data at Rest (experimental) +kube_encrypt_secret_data: false # DNS configuration. # Kubernetes cluster name, also will be used as DNS domain cluster_name: cluster.local # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods ndots: 2 -# Can be dnsmasq_kubedns, kubedns, manual or none +# Can be dnsmasq_kubedns, kubedns, coredns, coredns_dual, manual or none dns_mode: kubedns # Set manual server if using a custom cluster DNS server #manual_dns_server: 10.x.x.x @@ -129,21 +134,15 @@ resolvconf_mode: docker_dns deploy_netchecker: false # Ip address of the kubernetes skydns service skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" +skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}" dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" dns_domain: "{{ cluster_name }}" -# Path used to store Docker data -docker_daemon_graph: "/var/lib/docker" - -## A string of extra options to pass to the docker daemon. -## This string should be exactly as you wish it to appear. -## An obvious use case is allowing insecure-registry access -## to self hosted registries like so: +## Container runtime +## docker for docker and crio for cri-o. +container_manager: docker -docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }} {{ docker_log_opts }}" -docker_bin_dir: "/usr/bin" - -# Settings for containerized control plane (etcd/kubelet/secrets) +## Settings for containerized control plane (etcd/kubelet/secrets) etcd_deployment_type: docker kubelet_deployment_type: host vault_deployment_type: docker @@ -152,60 +151,15 @@ helm_deployment_type: host # K8s image pull policy (imagePullPolicy) k8s_image_pull_policy: IfNotPresent -# Kubernetes dashboard -# RBAC required. see docs/getting-started.md for access details. -dashboard_enabled: true - -# Monitoring apps for k8s -efk_enabled: false - -# Helm deployment -helm_enabled: false - -# Istio deployment -istio_enabled: false - -# Registry deployment -registry_enabled: false - -# Local volume provisioner deployment -local_volume_provisioner_enabled: false -# local_volume_provisioner_namespace: "{{ system_namespace }}" -local_volume_provisioner_base_dir: /mnt/disks -local_volume_provisioner_mount_dir: /mnt/disks -# local_volume_provisioner_storage_class: local-storage - -# CephFS provisioner deployment -cephfs_provisioner_enabled: false -# cephfs_provisioner_namespace: "{{ system_namespace }}" -# cephfs_provisioner_cluster: ceph -# cephfs_provisioner_monitors: -# - 172.24.0.1:6789 -# - 172.24.0.2:6789 -# - 172.24.0.3:6789 -# cephfs_provisioner_admin_id: admin -# cephfs_provisioner_secret: secret -# cephfs_provisioner_storage_class: cephfs - -# Nginx ingress controller deployment -ingress_nginx_enabled: false -# ingress_nginx_namespace: "ingress-nginx" -# ingress_nginx_insecure_port: 80 -# ingress_nginx_secure_port: 443 -# ingress_nginx_configmap: -# map-hash-bucket-size: "128" -# ssl-protocols: "SSLv2" -# ingress_nginx_configmap_tcp_services: -# 9000: "default/example-go:8080" -# ingress_nginx_configmap_udp_services: -# 53: "kube-system/kube-dns:53" +# audit log for kubernetes +kubernetes_audit: false -# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now ) -persistent_volumes_enabled: false +# pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled) +podsecuritypolicy_enabled: false -# Make a copy of kubeconfig on the host that runs Ansible in GITDIR/artifacts +# Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts # kubeconfig_localhost: false -# Download kubectl onto the host that runs Ansible in GITDIR/artifacts +# Download kubectl onto the host that runs Ansible in {{ bin_dir }} # kubectl_localhost: false # dnsmasq @@ -228,3 +182,20 @@ persistent_volumes_enabled: false ## See https://github.com/kubernetes-incubator/kubespray/issues/2141 ## Set this variable to true to get rid of this issue volume_cross_zone_attachment: false +# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now ) +persistent_volumes_enabled: false + +## Container Engine Acceleration +## Enable container accelertion feature, for example use gpu acceleration in containers +# nvidia_accelerator_enabled: true +## Nvidia GPU driver install. Install will by done by a (init) pod running as a daemonset. +## Important: if you use Ubuntu then you should set in all.yml 'docker_storage_options: -s overlay2' +## Array with nvida_gpu_nodes, leave empty or comment if you dont't want to install drivers. +## Labels and taints won't be set to nodes if they are not in the array. +# nvidia_gpu_nodes: +# - kube-gpu-001 +# nvidia_driver_version: "384.111" +## flavor can be tesla or gtx +# nvidia_gpu_flavor: gtx + + diff --git a/inventory/sample/hosts.ini b/inventory/sample/hosts.ini index 13cc3612e33..80c854d0a17 100644 --- a/inventory/sample/hosts.ini +++ b/inventory/sample/hosts.ini @@ -1,14 +1,15 @@ # ## Configure 'ip' variable to bind kubernetes services on a # ## different ip than the default iface -# node1 ansible_ssh_host=95.54.0.12 # ip=10.3.0.1 -# node2 ansible_ssh_host=95.54.0.13 # ip=10.3.0.2 -# node3 ansible_ssh_host=95.54.0.14 # ip=10.3.0.3 -# node4 ansible_ssh_host=95.54.0.15 # ip=10.3.0.4 -# node5 ansible_ssh_host=95.54.0.16 # ip=10.3.0.5 -# node6 ansible_ssh_host=95.54.0.17 # ip=10.3.0.6 +# ## We should set etcd_member_name for etcd cluster. The node that is not a etcd member do not need to set the value, or can set the empty string value. +# node1 ansible_host=95.54.0.12 # ip=10.3.0.1 etcd_member_name=etcd1 +# node2 ansible_host=95.54.0.13 # ip=10.3.0.2 etcd_member_name=etcd2 +# node3 ansible_host=95.54.0.14 # ip=10.3.0.3 etcd_member_name=etcd3 +# node4 ansible_host=95.54.0.15 # ip=10.3.0.4 etcd_member_name=etcd4 +# node5 ansible_host=95.54.0.16 # ip=10.3.0.5 etcd_member_name=etcd5 +# node6 ansible_host=95.54.0.17 # ip=10.3.0.6 etcd_member_name=etcd6 # ## configure a bastion host if your nodes are not directly reachable -# bastion ansible_ssh_host=x.x.x.x +# bastion ansible_host=x.x.x.x ansible_user=some_user # [kube-master] # node1 @@ -26,6 +27,6 @@ # node5 # node6 -# [k8s-cluster:children] -# kube-node -# kube-master +[k8s-cluster:children] +kube-master +kube-node diff --git a/remove-node.yml b/remove-node.yml index fbc5bc8ba6f..0fae1a99425 100644 --- a/remove-node.yml +++ b/remove-node.yml @@ -1,9 +1,11 @@ --- - hosts: all + vars: + ansible_ssh_pipelining: true gather_facts: true -- hosts: etcd:k8s-cluster:vault:calico-rr +- hosts: "{{ node | default('etcd:k8s-cluster:vault:calico-rr') }}" vars_prompt: name: "delete_nodes_confirmation" prompt: "Are you sure you want to delete nodes state? Type 'yes' to delete nodes." @@ -20,8 +22,9 @@ roles: - { role: remove-node/pre-remove, tags: pre-remove } -- hosts: kube-node +- hosts: "{{ node | default('kube-node') }}" roles: + - { role: kubespray-defaults } - { role: reset, tags: reset } - hosts: kube-master diff --git a/requirements.txt b/requirements.txt index 80ffd74ed51..01ff9f23cc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ -pbr>=1.6 ansible>=2.4.0 -netaddr jinja2>=2.9.6 +netaddr +pbr>=1.6 +ansible-modules-hashivault>=3.9.4 +hvac diff --git a/roles/adduser/tasks/main.yml b/roles/adduser/tasks/main.yml index 43ec8ebbb9a..3854ec4119a 100644 --- a/roles/adduser/tasks/main.yml +++ b/roles/adduser/tasks/main.yml @@ -7,7 +7,7 @@ - name: User | Create User user: comment: "{{user.comment|default(omit)}}" - createhome: "{{user.create_home|default(omit)}}" + createhome: "{{user.createhome|default(omit)}}" group: "{{user.group|default(user.name)}}" home: "{{user.home|default(omit)}}" shell: "{{user.shell|default(omit)}}" diff --git a/roles/bootstrap-os/defaults/main.yml b/roles/bootstrap-os/defaults/main.yml index c191ebd2b3c..5d2f7321a79 100644 --- a/roles/bootstrap-os/defaults/main.yml +++ b/roles/bootstrap-os/defaults/main.yml @@ -4,3 +4,6 @@ pip_python_coreos_modules: - six override_system_hostname: true + + +coreos_auto_upgrade: true diff --git a/roles/bootstrap-os/files/bootstrap.sh b/roles/bootstrap-os/files/bootstrap.sh index a2ad29b6c51..dbef6c8b103 100644 --- a/roles/bootstrap-os/files/bootstrap.sh +++ b/roles/bootstrap-os/files/bootstrap.sh @@ -18,7 +18,11 @@ mv -n pypy-$PYPY_VERSION-linux64 pypy ## library fixup mkdir -p pypy/lib -ln -snf /lib64/libncurses.so.5.9 $BINDIR/pypy/lib/libtinfo.so.5 +if [ -f /lib64/libncurses.so.5.9 ]; then + ln -snf /lib64/libncurses.so.5.9 $BINDIR/pypy/lib/libtinfo.so.5 +elif [ -f /lib64/libncurses.so.6.1 ]; then + ln -snf /lib64/libncurses.so.6.1 $BINDIR/pypy/lib/libtinfo.so.5 +fi cat > $BINDIR/python < 0) + when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic) and (docker_repo_info.repos|length > 0) + +- name: ensure docker-engine repository public key is installed + action: "{{ dockerproject_repo_key_info.pkg_key }}" + args: + id: "{{item}}" + url: "{{dockerproject_repo_key_info.url}}" + state: present + register: keyserver_task_result + until: keyserver_task_result|succeeded + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: "{{ dockerproject_repo_key_info.repo_keys }}" + when: + - not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic) + - use_docker_engine is defined and use_docker_engine + +- name: ensure docker-engine repository is enabled + action: "{{ dockerproject_repo_info.pkg_repo }}" + args: + repo: "{{item}}" + state: present + with_items: "{{ dockerproject_repo_info.repos }}" + when: + - use_docker_engine is defined and use_docker_engine + - not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic) and (dockerproject_repo_info.repos|length > 0) - name: Configure docker repository on RedHat/CentOS template: src: "rh_docker.repo.j2" - dest: "/etc/yum.repos.d/docker.repo" + dest: "{{ yum_repo_dir }}/docker.repo" when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic +- name: Copy yum.conf for editing + copy: + src: "{{ yum_conf }}" + dest: "{{ docker_yum_conf }}" + remote_src: yes + when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic + +- name: Edit copy of yum.conf to set obsoletes=0 + lineinfile: + path: "{{ docker_yum_conf }}" + state: present + regexp: '^obsoletes=' + line: 'obsoletes=0' + when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic + + - name: ensure docker packages are installed action: "{{ docker_package_info.pkg_mgr }}" args: pkg: "{{item.name}}" force: "{{item.force|default(omit)}}" + conf_file: "{{item.yum_conf|default(omit)}}" state: present + update_cache: yes register: docker_task_result until: docker_task_result|succeeded retries: 4 @@ -70,6 +126,32 @@ with_items: "{{ docker_package_info.pkgs }}" notify: restart docker when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) and (docker_package_info.pkgs|length > 0) + ignore_errors: true + +- name: get available packages on Ubuntu + command: apt-cache policy docker-ce + when: docker_task_result|failed + register: available_packages + +- name: show available packages on ubuntu + fail: + msg: "{{available_packages}}" + when: docker_task_result|failed + +# This is required to ensure any apt upgrade will not break kubernetes +- name: Set docker pin priority to apt_preferences on Debian family + template: + src: "apt_preferences.d/debian_docker.j2" + dest: "/etc/apt/preferences.d/docker" + owner: "root" + mode: 0644 + when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic) + +- name: ensure service is started if docker packages are already present + service: + name: docker + state: started + when: docker_task_result is not changed - name: flush handlers so we can wait for docker to come up meta: flush_handlers diff --git a/roles/docker/tasks/pre-upgrade.yml b/roles/docker/tasks/pre-upgrade.yml new file mode 100644 index 00000000000..9d5d27ef6c3 --- /dev/null +++ b/roles/docker/tasks/pre-upgrade.yml @@ -0,0 +1,32 @@ +--- +- name: Ensure old versions of Docker are not installed. | Debian + package: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-engine + - docker.io + when: + - ansible_os_family == 'Debian' + - (docker_versioned_pkg[docker_version | string] | search('docker-ce')) + +- name: Ensure old versions of Docker are not installed. | RedHat + package: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-common + - docker-engine + - docker-selinux + - docker-client + - docker-client-latest + - docker-latest + - docker-latest-logrotate + - docker-logrotate + - docker-engine-selinux + when: + - ansible_os_family == 'RedHat' + - (docker_versioned_pkg[docker_version | string] | search('docker-ce')) + - not is_atomic diff --git a/roles/docker/tasks/set_facts_dns.yml b/roles/docker/tasks/set_facts_dns.yml index 7152b442b18..8303e63ea9d 100644 --- a/roles/docker/tasks/set_facts_dns.yml +++ b/roles/docker/tasks/set_facts_dns.yml @@ -3,8 +3,10 @@ - name: set dns server for docker set_fact: docker_dns_servers: |- - {%- if dns_mode == 'kubedns' -%} + {%- if dns_mode in ['kubedns', 'coredns'] -%} {{ [ skydns_server ] }} + {%- elif dns_mode == 'coredns_dual' -%} + {{ [ skydns_server ] + [ skydns_server_secondary ] }} {%- elif dns_mode == 'dnsmasq_kubedns' -%} {{ [ dnsmasq_dns_server ] }} {%- elif dns_mode == 'manual' -%} @@ -24,7 +26,7 @@ - name: add upstream dns servers (only when dnsmasq is not used) set_fact: docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers|default([]) }}" - when: dns_mode == 'kubedns' + when: dns_mode in ['kubedns', 'coredns', 'coredns_dual'] - name: add global searchdomains set_fact: @@ -54,7 +56,7 @@ - name: check number of nameservers fail: - msg: "Too many nameservers. You can relax this check by set docker_dns_servers_strict=no and we will only use the first 3." + msg: "Too many nameservers. You can relax this check by set docker_dns_servers_strict=false in all.yml and we will only use the first 3." when: docker_dns_servers|length > 3 and docker_dns_servers_strict|bool - name: rtrim number of nameservers to 3 diff --git a/roles/docker/templates/apt_preferences.d/debian_docker.j2 b/roles/docker/templates/apt_preferences.d/debian_docker.j2 new file mode 100644 index 00000000000..f21008b6c14 --- /dev/null +++ b/roles/docker/templates/apt_preferences.d/debian_docker.j2 @@ -0,0 +1,3 @@ +Package: docker-ce +Pin: version {{ docker_version }}.* +Pin-Priority: 1001 \ No newline at end of file diff --git a/roles/docker/templates/docker-options.conf.j2 b/roles/docker/templates/docker-options.conf.j2 index c70f3d89fb8..296f5a8a10e 100644 --- a/roles/docker/templates/docker-options.conf.j2 +++ b/roles/docker/templates/docker-options.conf.j2 @@ -1,3 +1,5 @@ [Service] -Environment="DOCKER_OPTS={{ docker_options | default('') }} \ ---iptables=false" +Environment="DOCKER_OPTS={{ docker_options|default('') }} --iptables={{ docker_iptables_enabled | default('false') }}" +{% if docker_mount_flags is defined and docker_mount_flags != "" %} +MountFlags={{ docker_mount_flags }} +{% endif %} diff --git a/roles/docker/templates/docker.service.j2 b/roles/docker/templates/docker.service.j2 index 29abb6d53bb..8dc82bbb205 100644 --- a/roles/docker/templates/docker.service.j2 +++ b/roles/docker/templates/docker.service.j2 @@ -7,6 +7,9 @@ Wants=docker-storage-setup.service {% elif ansible_os_family == "Debian" %} After=network.target docker.socket Wants=docker.socket +{% elif ansible_os_family == "Suse" %} +After=network.target containerd.socket containerd.service +Requires=containerd.socket containerd.service {% endif %} [Service] @@ -19,6 +22,9 @@ ExecReload=/bin/kill -s HUP $MAINPID Delegate=yes KillMode=process ExecStart={{ docker_bin_dir }}/docker{% if installed_docker_version.stdout|version_compare('17.03', '<') %} daemon{% else %}d{% endif %} \ +{% if ansible_os_family == "Suse" %} + --containerd /run/containerd/containerd.sock --add-runtime oci=/usr/bin/docker-runc \ +{% endif %} $DOCKER_OPTS \ $DOCKER_STORAGE_OPTIONS \ $DOCKER_NETWORK_OPTIONS \ @@ -31,7 +37,10 @@ LimitNOFILE=1048576 LimitNPROC=1048576 LimitCORE=infinity TimeoutStartSec=1min -Restart=on-abnormal +# restart the docker process if it exits prematurely +Restart=on-failure +StartLimitBurst=3 +StartLimitInterval=60s [Install] WantedBy=multi-user.target diff --git a/roles/docker/templates/rh_docker.repo.j2 b/roles/docker/templates/rh_docker.repo.j2 index 7cb728625dd..fe2aeac1c2e 100644 --- a/roles/docker/templates/rh_docker.repo.j2 +++ b/roles/docker/templates/rh_docker.repo.j2 @@ -1,7 +1,15 @@ -[dockerrepo] -name=Docker Repository +[docker-ce] +name=Docker-CE Repository baseurl={{ docker_rh_repo_base_url }} enabled=1 gpgcheck=1 gpgkey={{ docker_rh_repo_gpgkey }} {% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %} + +[docker-engine] +name=Docker-Engine Repository +baseurl={{ dockerproject_rh_repo_base_url }} +enabled=1 +gpgcheck=1 +gpgkey={{ dockerproject_rh_repo_gpgkey }} +{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %} diff --git a/roles/docker/vars/debian.yml b/roles/docker/vars/debian.yml index 0a43c7c79ea..8138996c10b 100644 --- a/roles/docker/vars/debian.yml +++ b/roles/docker/vars/debian.yml @@ -2,12 +2,14 @@ docker_kernel_min_version: '3.10' # https://download.docker.com/linux/debian/ +# https://apt.dockerproject.org/repo/dists/debian-wheezy/main/filelist docker_versioned_pkg: 'latest': docker-ce '1.11': docker-engine=1.11.2-0~{{ ansible_distribution_release|lower }} '1.12': docker-engine=1.12.6-0~debian-{{ ansible_distribution_release|lower }} '1.13': docker-engine=1.13.1-0~debian-{{ ansible_distribution_release|lower }} '17.03': docker-ce=17.03.2~ce-0~debian-{{ ansible_distribution_release|lower }} + '17.09': docker-ce=17.09.0~ce-0~debian-{{ ansible_distribution_release|lower }} 'stable': docker-ce=17.03.2~ce-0~debian-{{ ansible_distribution_release|lower }} 'edge': docker-ce=17.12.1~ce-0~debian-{{ ansible_distribution_release|lower }} @@ -30,3 +32,17 @@ docker_repo_info: deb {{ docker_debian_repo_base_url }} {{ ansible_distribution_release|lower }} stable + +dockerproject_repo_key_info: + pkg_key: apt_key + url: '{{ dockerproject_apt_repo_gpgkey }}' + repo_keys: + - 58118E89F3A912897C070ADBF76221572C52609D + +dockerproject_repo_info: + pkg_repo: apt_repository + repos: + - > + deb {{ dockerproject_apt_repo_base_url }} + {{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }} + main diff --git a/roles/docker/vars/redhat-aarch64.yml b/roles/docker/vars/redhat-aarch64.yml new file mode 100644 index 00000000000..0bad0593de5 --- /dev/null +++ b/roles/docker/vars/redhat-aarch64.yml @@ -0,0 +1,28 @@ +--- +docker_kernel_min_version: '0' + +# overide defaults, missing 17.03 for aarch64 +docker_version: '1.13' + +# http://mirror.centos.org/altarch/7/extras/aarch64/Packages/ +# or do 'yum --showduplicates list docker' +docker_versioned_pkg: + 'latest': docker + '1.12': docker-1.12.6-48.git0fdc778.el7 + '1.13': docker-1.13.1-63.git94f4240.el7 + +# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package +# http://mirror.centos.org/altarch/7/extras/aarch64/Packages/ + +docker_package_info: + pkg_mgr: yum + pkgs: + - name: "{{ docker_versioned_pkg[docker_version | string] }}" + +docker_repo_key_info: + pkg_key: '' + repo_keys: [] + +docker_repo_info: + pkg_repo: '' + repos: [] diff --git a/roles/docker/vars/redhat.yml b/roles/docker/vars/redhat.yml index 96950719e9c..57970eb50e7 100644 --- a/roles/docker/vars/redhat.yml +++ b/roles/docker/vars/redhat.yml @@ -3,6 +3,7 @@ docker_kernel_min_version: '0' # https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package # https://download.docker.com/linux/centos/7/x86_64/stable/Packages/ +# https://yum.dockerproject.org/repo/main/centos/7 # or do 'yum --showduplicates list docker-engine' docker_versioned_pkg: 'latest': docker-ce @@ -10,11 +11,15 @@ docker_versioned_pkg: '1.12': docker-engine-1.12.6-1.el7.centos '1.13': docker-engine-1.13.1-1.el7.centos '17.03': docker-ce-17.03.2.ce-1.el7.centos + '17.09': docker-ce-17.09.0.ce-1.el7.centos 'stable': docker-ce-17.03.2.ce-1.el7.centos - 'edge': docker-ce-17.03.2.ce-1.el7.centos + 'edge': docker-ce-17.12.1.ce-1.el7.centos docker_selinux_versioned_pkg: 'latest': docker-ce-selinux + '1.11': docker-engine-selinux-1.11.2-1.el7.centos + '1.12': docker-engine-selinux-1.12.6-1.el7.centos + '1.13': docker-engine-selinux-1.13.1-1.el7.centos '17.03': docker-ce-selinux-17.03.2.ce-1.el7.centos 'stable': docker-ce-selinux-17.03.2.ce-1.el7.centos 'edge': docker-ce-selinux-17.03.2.ce-1.el7.centos @@ -24,7 +29,9 @@ docker_package_info: pkg_mgr: yum pkgs: - name: "{{ docker_selinux_versioned_pkg[docker_selinux_version | string] }}" + yum_conf: "{{ docker_yum_conf }}" - name: "{{ docker_versioned_pkg[docker_version | string] }}" + yum_conf: "{{ docker_yum_conf }}" docker_repo_key_info: pkg_key: '' diff --git a/roles/docker/vars/suse.yml b/roles/docker/vars/suse.yml new file mode 100644 index 00000000000..d89a50a7f4b --- /dev/null +++ b/roles/docker/vars/suse.yml @@ -0,0 +1,15 @@ +--- +docker_kernel_min_version: '0' + +docker_package_info: + pkg_mgr: zypper + pkgs: + - name: docker + +docker_repo_key_info: + pkg_key: '' + repo_keys: [] + +docker_repo_info: + pkg_repo: '' + repos: [] diff --git a/roles/docker/vars/ubuntu-amd64.yml b/roles/docker/vars/ubuntu-amd64.yml new file mode 100644 index 00000000000..b88800b4c4f --- /dev/null +++ b/roles/docker/vars/ubuntu-amd64.yml @@ -0,0 +1,49 @@ +--- +docker_kernel_min_version: '3.10' + +# https://download.docker.com/linux/ubuntu/ +docker_versioned_pkg: + 'latest': docker-ce + '1.11': docker-engine=1.11.2-0~{{ ansible_distribution_release|lower }} + '1.12': docker-engine=1.12.6-0~ubuntu-{{ ansible_distribution_release|lower }} + '1.13': docker-engine=1.13.1-0~ubuntu-{{ ansible_distribution_release|lower }} + '17.03': docker-ce=17.03.2~ce-0~ubuntu-{{ ansible_distribution_release|lower }} + '17.09': docker-ce=17.09.0~ce-0~ubuntu-{{ ansible_distribution_release|lower }} + '17.12': docker-ce=17.12.1~ce-0~ubuntu-{{ ansible_distribution_release|lower }} + '18.06': docker-ce=18.06.1~ce~3-0~ubuntu + 'stable': docker-ce=17.03.2~ce-0~ubuntu-{{ ansible_distribution_release|lower }} + 'edge': docker-ce=18.06.1~ce~3-0~ubuntu + +docker_package_info: + pkg_mgr: apt + pkgs: + - name: "{{ docker_versioned_pkg[docker_version | string] }}" + force: yes + +docker_repo_key_info: + pkg_key: apt_key + url: '{{ docker_ubuntu_repo_gpgkey }}' + repo_keys: + - 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + +docker_repo_info: + pkg_repo: apt_repository + repos: + - > + deb {{ docker_ubuntu_repo_base_url }} + {{ ansible_distribution_release|lower }} + stable + +dockerproject_repo_key_info: + pkg_key: apt_key + url: '{{ dockerproject_apt_repo_gpgkey }}' + repo_keys: + - 58118E89F3A912897C070ADBF76221572C52609D + +dockerproject_repo_info: + pkg_repo: apt_repository + repos: + - > + deb {{ dockerproject_apt_repo_base_url }} + {{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }} + main diff --git a/roles/docker/vars/ubuntu.yml b/roles/docker/vars/ubuntu-arm64.yml similarity index 54% rename from roles/docker/vars/ubuntu.yml rename to roles/docker/vars/ubuntu-arm64.yml index 897c23ce003..3dbb1a0e4f4 100644 --- a/roles/docker/vars/ubuntu.yml +++ b/roles/docker/vars/ubuntu-arm64.yml @@ -1,13 +1,12 @@ --- docker_kernel_min_version: '3.10' - +docker_version: 17.09 # https://download.docker.com/linux/ubuntu/ docker_versioned_pkg: 'latest': docker-ce - '1.11': docker-engine=1.11.1-0~{{ ansible_distribution_release|lower }} - '1.12': docker-engine=1.12.6-0~ubuntu-{{ ansible_distribution_release|lower }} - '1.13': docker-engine=1.13.1-0~ubuntu-{{ ansible_distribution_release|lower }} - '17.03': docker-ce=17.03.2~ce-0~ubuntu-{{ ansible_distribution_release|lower }} + '17.09': docker-ce=17.09.1~ce-0~ubuntu + 'stable': docker-ce=17.09.1~ce-0~ubuntu + 'edge': docker-ce=17.12.1~ce-0~ubuntu docker_package_info: pkg_mgr: apt @@ -28,3 +27,17 @@ docker_repo_info: deb {{ docker_ubuntu_repo_base_url }} {{ ansible_distribution_release|lower }} stable + +dockerproject_repo_key_info: + pkg_key: apt_key + url: '{{ dockerproject_apt_repo_gpgkey }}' + repo_keys: + - 58118E89F3A912897C070ADBF76221572C52609D + +dockerproject_repo_info: + pkg_repo: apt_repository + repos: + - > + deb {{ dockerproject_apt_repo_base_url }} + {{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }} + main diff --git a/roles/docker/vars/ubuntu-bionic.yml b/roles/docker/vars/ubuntu-bionic.yml new file mode 100644 index 00000000000..3f1fa900012 --- /dev/null +++ b/roles/docker/vars/ubuntu-bionic.yml @@ -0,0 +1,32 @@ +--- +docker_kernel_min_version: '3.10' + +use_docker_engine: false + +docker_versioned_pkg: + 'latest': docker-ce + '17.03': docker-ce=17.03.2~ce-0~ubuntu-xenial + '18.03': docker-ce=18.03.1~ce-3-0~ubuntu + '18.06': docker-ce=18.06.1~ce~3-0~ubuntu + 'stable': docker-ce=18.06.1~ce~3-0~ubuntu + 'edge': docker-ce=18.06.1~ce~3-0~ubuntu + +docker_package_info: + pkg_mgr: apt + pkgs: + - name: "{{ docker_versioned_pkg[docker_version | string] }}" + force: yes + +docker_repo_key_info: + pkg_key: apt_key + url: '{{ docker_ubuntu_repo_gpgkey }}' + repo_keys: + - 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + +docker_repo_info: + pkg_repo: apt_repository + repos: + - > + deb [arch=amd64] {{ docker_ubuntu_repo_base_url }} + xenial + stable \ No newline at end of file diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index d87f4b9239c..c27224b14ff 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -1,5 +1,5 @@ --- -local_release_dir: /tmp +local_release_dir: /tmp/releases # Used to only evaluate vars from download role skip_downloads: false @@ -10,6 +10,9 @@ skip_downloads: false download_run_once: False download_compress: 1 +# if this is set to true will download container +download_container: True + # if this is set to true, uses the localhost for download_run_once mode # (requires docker and sudo to access docker). You may want this option for # local caching of docker images or for Container Linux by CoreOS cluster nodes. @@ -23,39 +26,50 @@ download_always_pull: False # Use the first kube-master if download_localhost is not set download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}" +# Arch of Docker images and needed packages +image_arch: "{{host_architecture | default('amd64')}}" + # Versions -kube_version: v1.9.3 +kube_version: v1.11.3 kubeadm_version: "{{ kube_version }}" -etcd_version: v3.2.4 +etcd_version: v3.2.18 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v2.6.2" -calico_ctl_version: "v1.6.1" -calico_cni_version: "v1.11.0" -calico_policy_version: "v1.0.0" -calico_rr_version: "v0.4.0" -flannel_version: "v0.9.1" +calico_version: "v3.1.3" +calico_ctl_version: "v3.1.3" +calico_cni_version: "v3.1.3" +calico_policy_version: "v3.1.3" +calico_rr_version: "v0.6.1" +flannel_version: "v0.10.0" flannel_cni_version: "v0.3.0" -istio_version: "0.2.6" -vault_version: 0.8.1 -weave_version: 2.2.0 -pod_infra_version: 3.0 +vault_version: 0.10.1 +weave_version: "2.4.0" +pod_infra_version: 3.1 contiv_version: 1.1.7 -cilium_version: "v1.0.0-rc4" +cilium_version: "v1.2.0" # Download URLs -istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux" -kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm" -vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" +kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm" +vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_{{ image_arch }}.zip" +etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz" +hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64/hyperkube" # Checksums -istioctl_checksum: fd703063c540b8c0ab943f478c05ab257d88ae27224c746a27d0526ddbf7c370 -kubeadm_checksum: 9ebbb1fbf3a9e72d7df3f0dc02500dc8f957f39489b22cf577498c8a7c6b39b1 +etcd_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097 +hyperkube_checksum: dac8da16dd6688e52b5dc510f5dd0a20b54350d52fb27ceba2f018ba2c8be692 +kubeadm_checksum: 422a7a32ed9a7b1eaa2a4f9d121674dfbe80eb41e206092c13017d097f75aaec vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 # Containers +# In some cases, we need a way to set --registry-mirror or --insecure-registry for docker, +# it helps a lot for local private development or bare metal environment. +# So you need define --registry-mirror or --insecure-registry, and modify the following url address. +# example: +# You need to deploy kubernetes cluster on local private development. +# Also provide the address of your own private registry. +# And use --insecure-registry options for docker etcd_image_repo: "quay.io/coreos/etcd" -etcd_image_tag: "{{ etcd_version }}" +etcd_image_tag: "{{ etcd_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}" flannel_image_repo: "quay.io/coreos/flannel" flannel_image_tag: "{{ flannel_version }}" flannel_cni_image_repo: "quay.io/coreos/flannel-cni" @@ -70,16 +84,16 @@ calico_policy_image_repo: "quay.io/calico/kube-controllers" calico_policy_image_tag: "{{ calico_policy_version }}" calico_rr_image_repo: "quay.io/calico/routereflector" calico_rr_image_tag: "{{ calico_rr_version }}" -hyperkube_image_repo: "quay.io/coreos/hyperkube" -hyperkube_image_tag: "{{ kube_version }}_coreos.0" -pod_infra_image_repo: "gcr.io/google_containers/pause-amd64" +hyperkube_image_repo: "gcr.io/google-containers/hyperkube-{{ image_arch }}" +hyperkube_image_tag: "{{ kube_version }}" +pod_infra_image_repo: "gcr.io/google_containers/pause-{{ image_arch }}" pod_infra_image_tag: "{{ pod_infra_version }}" install_socat_image_repo: "xueshanf/install-socat" install_socat_image_tag: "latest" -netcheck_version: "v1.0" -netcheck_agent_img_repo: "quay.io/l23network/k8s-netchecker-agent" +netcheck_version: "v1.2.2" +netcheck_agent_img_repo: "mirantis/k8s-netchecker-agent" netcheck_agent_tag: "{{ netcheck_version }}" -netcheck_server_img_repo: "quay.io/l23network/k8s-netchecker-server" +netcheck_server_img_repo: "mirantis/k8s-netchecker-server" netcheck_server_tag: "{{ netcheck_version }}" weave_kube_image_repo: "weaveworks/weave-kube" weave_kube_image_tag: "{{ weave_version }}" @@ -89,46 +103,67 @@ contiv_image_repo: "contiv/netplugin" contiv_image_tag: "{{ contiv_version }}" contiv_auth_proxy_image_repo: "contiv/auth_proxy" contiv_auth_proxy_image_tag: "{{ contiv_version }}" +contiv_etcd_init_image_repo: "ferest/etcd-initer" +contiv_etcd_init_image_tag: latest cilium_image_repo: "docker.io/cilium/cilium" cilium_image_tag: "{{ cilium_version }}" - nginx_image_repo: nginx nginx_image_tag: 1.13 dnsmasq_version: 2.78 dnsmasq_image_repo: "andyshinn/dnsmasq" dnsmasq_image_tag: "{{ dnsmasq_version }}" -kubedns_version: 1.14.8 -kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-amd64" +kubedns_version: 1.14.10 +kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-{{ image_arch }}" kubedns_image_tag: "{{ kubedns_version }}" -dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64" + +coredns_version: "{%- if image_arch != 'amd64' -%}1.1.3{%- else -%}1.2.2{%- endif -%}" +coredns_image_repo: "gcr.io/google-containers/coredns" +coredns_image_tag: "{{ coredns_version }}{%- if image_arch != 'amd64' -%}__{{ image_arch}}_linux{%- endif -%}" + + +dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-{{ image_arch }}" dnsmasq_nanny_image_tag: "{{ kubedns_version }}" -dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-amd64" +dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-{{ image_arch }}" dnsmasq_sidecar_image_tag: "{{ kubedns_version }}" dnsmasqautoscaler_version: 1.1.2 -dnsmasqautoscaler_image_repo: "gcr.io/google_containers/cluster-proportional-autoscaler-amd64" +dnsmasqautoscaler_image_repo: "gcr.io/google_containers/cluster-proportional-autoscaler-{{ image_arch }}" dnsmasqautoscaler_image_tag: "{{ dnsmasqautoscaler_version }}" kubednsautoscaler_version: 1.1.2 -kubednsautoscaler_image_repo: "gcr.io/google_containers/cluster-proportional-autoscaler-amd64" +kubednsautoscaler_image_repo: "gcr.io/google_containers/cluster-proportional-autoscaler-{{ image_arch }}" kubednsautoscaler_image_tag: "{{ kubednsautoscaler_version }}" test_image_repo: busybox test_image_tag: latest -elasticsearch_version: "v2.4.1" -elasticsearch_image_repo: "gcr.io/google_containers/elasticsearch" +elasticsearch_version: "v5.6.4" +elasticsearch_image_repo: "k8s.gcr.io/elasticsearch" elasticsearch_image_tag: "{{ elasticsearch_version }}" -fluentd_version: "1.22" -fluentd_image_repo: "gcr.io/google_containers/fluentd-elasticsearch" +fluentd_version: "v2.0.4" +fluentd_image_repo: "k8s.gcr.io/fluentd-elasticsearch" fluentd_image_tag: "{{ fluentd_version }}" -kibana_version: "v4.6.1" -kibana_image_repo: "gcr.io/google_containers/kibana" +kibana_version: "5.6.4" +kibana_image_repo: "docker.elastic.co/kibana/kibana" kibana_image_tag: "{{ kibana_version }}" - -helm_version: "v2.8.1" +helm_version: "v2.9.1" helm_image_repo: "lachlanevenson/k8s-helm" helm_image_tag: "{{ helm_version }}" tiller_image_repo: "gcr.io/kubernetes-helm/tiller" tiller_image_tag: "{{ helm_version }}" vault_image_repo: "vault" vault_image_tag: "{{ vault_version }}" +registry_image_repo: "registry" +registry_image_tag: "2.6" +registry_proxy_image_repo: "gcr.io/google_containers/kube-registry-proxy" +registry_proxy_image_tag: "0.4" +local_volume_provisioner_image_repo: "quay.io/external_storage/local-volume-provisioner" +local_volume_provisioner_image_tag: "v2.1.0" +cephfs_provisioner_image_repo: "quay.io/external_storage/cephfs-provisioner" +cephfs_provisioner_image_tag: "v2.1.0-k8s1.11" +ingress_nginx_controller_image_repo: "quay.io/kubernetes-ingress-controller/nginx-ingress-controller" +ingress_nginx_controller_image_tag: "0.19.0" +ingress_nginx_default_backend_image_repo: "gcr.io/google_containers/defaultbackend" +ingress_nginx_default_backend_image_tag: "1.4" +cert_manager_version: "v0.4.1" +cert_manager_controller_image_repo: "quay.io/jetstack/cert-manager-controller" +cert_manager_controller_image_tag: "{{ cert_manager_version }}" downloads: netcheck_server: @@ -137,18 +172,37 @@ downloads: repo: "{{ netcheck_server_img_repo }}" tag: "{{ netcheck_server_tag }}" sha256: "{{ netcheck_server_digest_checksum|default(None) }}" + groups: + - k8s-cluster netcheck_agent: enabled: "{{ deploy_netchecker }}" container: true repo: "{{ netcheck_agent_img_repo }}" tag: "{{ netcheck_agent_tag }}" sha256: "{{ netcheck_agent_digest_checksum|default(None) }}" + groups: + - k8s-cluster etcd: enabled: true container: true repo: "{{ etcd_image_repo }}" tag: "{{ etcd_image_tag }}" sha256: "{{ etcd_digest_checksum|default(None) }}" + groups: + - etcd + etcd_file: + enabled: true + file: true + version: "{{ etcd_version }}" + dest: "etcd-{{ etcd_version }}-linux-amd64.tar.gz" + sha256: "{{ etcd_checksum }}" + source_url: "{{ etcd_download_url }}" + url: "{{ etcd_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - etcd kubeadm: enabled: "{{ kubeadm_enabled }}" file: true @@ -160,145 +214,207 @@ downloads: unarchive: false owner: "root" mode: "0755" - istioctl: - enabled: "{{ istio_enabled }}" - file: true - version: "{{ istio_version }}" - dest: "istio/istioctl" - sha256: "{{ istioctl_checksum }}" - source_url: "{{ istioctl_download_url }}" - url: "{{ istioctl_download_url }}" - unarchive: false - owner: "root" - mode: "0755" + groups: + - k8s-cluster hyperkube: enabled: true container: true repo: "{{ hyperkube_image_repo }}" tag: "{{ hyperkube_image_tag }}" sha256: "{{ hyperkube_digest_checksum|default(None) }}" + groups: + - k8s-cluster + hyperkube_file: + enabled: true + file: true + version: "{{ kube_version }}" + dest: "hyperkube" + sha256: "{{ hyperkube_checksum }}" + source_url: "{{ hyperkube_download_url }}" + url: "{{ hyperkube_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s-cluster cilium: enabled: "{{ kube_network_plugin == 'cilium' }}" container: true repo: "{{ cilium_image_repo }}" tag: "{{ cilium_image_tag }}" sha256: "{{ cilium_digest_checksum|default(None) }}" + groups: + - k8s-cluster flannel: enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}" container: true repo: "{{ flannel_image_repo }}" tag: "{{ flannel_image_tag }}" sha256: "{{ flannel_digest_checksum|default(None) }}" + groups: + - k8s-cluster flannel_cni: enabled: "{{ kube_network_plugin == 'flannel' }}" container: true repo: "{{ flannel_cni_image_repo }}" tag: "{{ flannel_cni_image_tag }}" sha256: "{{ flannel_cni_digest_checksum|default(None) }}" + groups: + - k8s-cluster calicoctl: enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}" container: true repo: "{{ calicoctl_image_repo }}" tag: "{{ calicoctl_image_tag }}" sha256: "{{ calicoctl_digest_checksum|default(None) }}" + groups: + - k8s-cluster calico_node: enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}" container: true repo: "{{ calico_node_image_repo }}" tag: "{{ calico_node_image_tag }}" sha256: "{{ calico_node_digest_checksum|default(None) }}" + groups: + - k8s-cluster calico_cni: enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}" container: true repo: "{{ calico_cni_image_repo }}" tag: "{{ calico_cni_image_tag }}" sha256: "{{ calico_cni_digest_checksum|default(None) }}" + groups: + - k8s-cluster calico_policy: enabled: "{{ enable_network_policy or kube_network_plugin == 'canal' }}" container: true repo: "{{ calico_policy_image_repo }}" tag: "{{ calico_policy_image_tag }}" sha256: "{{ calico_policy_digest_checksum|default(None) }}" + groups: + - k8s-cluster calico_rr: enabled: "{{ peer_with_calico_rr is defined and peer_with_calico_rr and kube_network_plugin == 'calico' }}" container: true repo: "{{ calico_rr_image_repo }}" tag: "{{ calico_rr_image_tag }}" sha256: "{{ calico_rr_digest_checksum|default(None) }}" + groups: + - calico-rr weave_kube: enabled: "{{ kube_network_plugin == 'weave' }}" container: true repo: "{{ weave_kube_image_repo }}" tag: "{{ weave_kube_image_tag }}" sha256: "{{ weave_kube_digest_checksum|default(None) }}" + groups: + - k8s-cluster weave_npc: enabled: "{{ kube_network_plugin == 'weave' }}" container: true repo: "{{ weave_npc_image_repo }}" tag: "{{ weave_npc_image_tag }}" sha256: "{{ weave_npc_digest_checksum|default(None) }}" + groups: + - k8s-cluster contiv: enabled: "{{ kube_network_plugin == 'contiv' }}" container: true repo: "{{ contiv_image_repo }}" tag: "{{ contiv_image_tag }}" sha256: "{{ contiv_digest_checksum|default(None) }}" + groups: + - k8s-cluster contiv_auth_proxy: enabled: "{{ kube_network_plugin == 'contiv' }}" container: true repo: "{{ contiv_auth_proxy_image_repo }}" tag: "{{ contiv_auth_proxy_image_tag }}" sha256: "{{ contiv_auth_proxy_digest_checksum|default(None) }}" + groups: + - k8s-cluster + contiv_etcd_init: + enabled: "{{ kube_network_plugin == 'contiv' }}" + container: true + repo: "{{ contiv_etcd_init_image_repo }}" + tag: "{{ contiv_etcd_init_image_tag }}" + sha256: "{{ contiv_etcd_init_digest_checksum|default(None) }}" + groups: + - k8s-cluster pod_infra: enabled: true container: true repo: "{{ pod_infra_image_repo }}" tag: "{{ pod_infra_image_tag }}" sha256: "{{ pod_infra_digest_checksum|default(None) }}" + groups: + - k8s-cluster install_socat: enabled: "{{ ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] }}" container: true repo: "{{ install_socat_image_repo }}" tag: "{{ install_socat_image_tag }}" sha256: "{{ install_socat_digest_checksum|default(None) }}" + groups: + - k8s-cluster nginx: - enabled: true + enabled: "{{ loadbalancer_apiserver_localhost }}" container: true repo: "{{ nginx_image_repo }}" tag: "{{ nginx_image_tag }}" sha256: "{{ nginx_digest_checksum|default(None) }}" + groups: + - kube-node dnsmasq: enabled: "{{ dns_mode == 'dnsmasq_kubedns' }}" container: true repo: "{{ dnsmasq_image_repo }}" tag: "{{ dnsmasq_image_tag }}" sha256: "{{ dnsmasq_digest_checksum|default(None) }}" + groups: + - kube-node kubedns: - enabled: true + enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true repo: "{{ kubedns_image_repo }}" tag: "{{ kubedns_image_tag }}" sha256: "{{ kubedns_digest_checksum|default(None) }}" + groups: + - kube-node + coredns: + enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] }}" + container: true + repo: "{{ coredns_image_repo }}" + tag: "{{ coredns_image_tag }}" + sha256: "{{ coredns_digest_checksum|default(None) }}" + groups: + - kube-node dnsmasq_nanny: - enabled: true + enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true repo: "{{ dnsmasq_nanny_image_repo }}" tag: "{{ dnsmasq_nanny_image_tag }}" sha256: "{{ dnsmasq_nanny_digest_checksum|default(None) }}" + groups: + - kube-node dnsmasq_sidecar: - enabled: true + enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true repo: "{{ dnsmasq_sidecar_image_repo }}" tag: "{{ dnsmasq_sidecar_image_tag }}" sha256: "{{ dnsmasq_sidecar_digest_checksum|default(None) }}" + groups: + - kube-node kubednsautoscaler: - enabled: true + enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true repo: "{{ kubednsautoscaler_image_repo }}" tag: "{{ kubednsautoscaler_image_tag }}" sha256: "{{ kubednsautoscaler_digest_checksum|default(None) }}" + groups: + - kube-node testbox: - enabled: true + enabled: false container: true repo: "{{ test_image_repo }}" tag: "{{ test_image_tag }}" @@ -309,35 +425,45 @@ downloads: repo: "{{ elasticsearch_image_repo }}" tag: "{{ elasticsearch_image_tag }}" sha256: "{{ elasticsearch_digest_checksum|default(None) }}" + groups: + - kube-node fluentd: enabled: "{{ efk_enabled }}" container: true repo: "{{ fluentd_image_repo }}" tag: "{{ fluentd_image_tag }}" sha256: "{{ fluentd_digest_checksum|default(None) }}" + groups: + - kube-node kibana: enabled: "{{ efk_enabled }}" container: true repo: "{{ kibana_image_repo }}" tag: "{{ kibana_image_tag }}" sha256: "{{ kibana_digest_checksum|default(None) }}" + groups: + - kube-node helm: enabled: "{{ helm_enabled }}" container: true repo: "{{ helm_image_repo }}" tag: "{{ helm_image_tag }}" sha256: "{{ helm_digest_checksum|default(None) }}" + groups: + - kube-node tiller: enabled: "{{ helm_enabled }}" container: true repo: "{{ tiller_image_repo }}" tag: "{{ tiller_image_tag }}" sha256: "{{ tiller_digest_checksum|default(None) }}" + groups: + - kube-node vault: enabled: "{{ cert_management == 'vault' }}" container: "{{ vault_deployment_type != 'host' }}" file: "{{ vault_deployment_type == 'host' }}" - dest: "vault/vault_{{ vault_version }}_linux_amd64.zip" + dest: "vault/vault_{{ vault_version }}_linux_{{ image_arch }}.zip" mode: "0755" owner: "vault" repo: "{{ vault_image_repo }}" @@ -347,6 +473,64 @@ downloads: unarchive: true url: "{{ vault_download_url }}" version: "{{ vault_version }}" + groups: + - vault + registry: + enabled: "{{ registry_enabled }}" + container: true + repo: "{{ registry_image_repo }}" + tag: "{{ registry_image_tag }}" + sha256: "{{ registry_digest_checksum|default(None) }}" + groups: + - kube-node + registry_proxy: + enabled: "{{ registry_enabled }}" + container: true + repo: "{{ registry_proxy_image_repo }}" + tag: "{{ registry_proxy_image_tag }}" + sha256: "{{ registry_proxy_digest_checksum|default(None) }}" + groups: + - kube-node + local_volume_provisioner: + enabled: "{{ local_volume_provisioner_enabled }}" + container: true + repo: "{{ local_volume_provisioner_image_repo }}" + tag: "{{ local_volume_provisioner_image_tag }}" + sha256: "{{ local_volume_provisioner_digest_checksum|default(None) }}" + groups: + - kube-node + cephfs_provisioner: + enabled: "{{ cephfs_provisioner_enabled }}" + container: true + repo: "{{ cephfs_provisioner_image_repo }}" + tag: "{{ cephfs_provisioner_image_tag }}" + sha256: "{{ cephfs_provisioner_digest_checksum|default(None) }}" + groups: + - kube-node + ingress_nginx_controller: + enabled: "{{ ingress_nginx_enabled }}" + container: true + repo: "{{ ingress_nginx_controller_image_repo }}" + tag: "{{ ingress_nginx_controller_image_tag }}" + sha256: "{{ ingress_nginx_controller_digest_checksum|default(None) }}" + groups: + - kube-node + ingress_nginx_default_backend: + enabled: "{{ ingress_nginx_enabled }}" + container: true + repo: "{{ ingress_nginx_default_backend_image_repo }}" + tag: "{{ ingress_nginx_default_backend_image_tag }}" + sha256: "{{ ingress_nginx_default_backend_digest_checksum|default(None) }}" + groups: + - kube-node + cert_manager_controller: + enabled: "{{ cert_manager_enabled }}" + container: true + repo: "{{ cert_manager_controller_image_repo }}" + tag: "{{ cert_manager_controller_image_tag }}" + sha256: "{{ cert_manager_controller_digest_checksum|default(None) }}" + groups: + - kube-node download_defaults: container: false diff --git a/roles/download/tasks/download_container.yml b/roles/download/tasks/download_container.yml index bbf7cec85f3..7e3923606fb 100644 --- a/roles/download/tasks/download_container.yml +++ b/roles/download/tasks/download_container.yml @@ -2,7 +2,7 @@ - name: container_download | Make download decision if pull is required by tag or sha256 include_tasks: set_docker_image_facts.yml delegate_to: "{{ download_delegate if download_run_once or omit }}" - delegate_facts: no + delegate_facts: yes run_once: "{{ download_run_once }}" when: - download.enabled @@ -38,3 +38,4 @@ - download.enabled - download.container - pull_required|default(download_always_pull) + - group_names | intersect(download.groups) | length diff --git a/roles/download/tasks/download_file.yml b/roles/download/tasks/download_file.yml index 664fa472858..832fec41ea0 100644 --- a/roles/download/tasks/download_file.yml +++ b/roles/download/tasks/download_file.yml @@ -13,6 +13,7 @@ when: - download.enabled - download.file + - group_names | intersect(download.groups) | length - name: file_download | Download item get_url: @@ -28,6 +29,7 @@ when: - download.enabled - download.file + - group_names | intersect(download.groups) | length - name: file_download | Extract archives unarchive: @@ -40,3 +42,4 @@ - download.enabled - download.file - download.unarchive|default(False) + - group_names | intersect(download.groups) | length diff --git a/roles/download/tasks/download_prep.yml b/roles/download/tasks/download_prep.yml index 1fd7abf2fb2..b44da45dae7 100644 --- a/roles/download/tasks/download_prep.yml +++ b/roles/download/tasks/download_prep.yml @@ -7,6 +7,7 @@ failed_when: false changed_when: false check_mode: no + when: download_container - name: container_download | Create dest directory for saved/loaded container images file: @@ -15,6 +16,7 @@ recurse: yes mode: 0755 owner: "{{ansible_ssh_user|default(ansible_user_id)}}" + when: download_container - name: container_download | create local directory for saved/loaded container images file: @@ -28,5 +30,6 @@ when: - download_run_once - download_delegate == 'localhost' + - download_container tags: - localhost diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index c6e910e5df5..1984f626d70 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -11,6 +11,7 @@ when: - not skip_downloads|default(false) - item.value.enabled + - (not (item.value.container|default(False))) or (item.value.container and download_container) - name: "Sync container" include_tasks: sync_container.yml @@ -20,5 +21,6 @@ when: - not skip_downloads|default(false) - item.value.enabled - - item.value.container + - "{{ item.value.container | default(False) }}" - download_run_once + - group_names | intersect(download.groups) | length diff --git a/roles/download/tasks/set_docker_image_facts.yml b/roles/download/tasks/set_docker_image_facts.yml index 7a9b73e38b5..87a73cfbd71 100644 --- a/roles/download/tasks/set_docker_image_facts.yml +++ b/roles/download/tasks/set_docker_image_facts.yml @@ -9,7 +9,7 @@ - name: Register docker images info raw: >- - {{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} (index .RepoTags 0) {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}" | tr '\n' ',' + {{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} if .RepoTags {{ '}}' }}{{ '{{' }} (index .RepoTags 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}{{ '{{' }} if .RepoDigests {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}" | tr '\n' ',' no_log: true register: docker_images failed_when: false diff --git a/roles/download/tasks/sync_container.yml b/roles/download/tasks/sync_container.yml index a15f78cde44..c7e37d7f3d6 100644 --- a/roles/download/tasks/sync_container.yml +++ b/roles/download/tasks/sync_container.yml @@ -17,6 +17,7 @@ - download.enabled - download.container - download_run_once + tags: - facts @@ -42,7 +43,7 @@ - name: container_download | Stat saved container image stat: - path: "{{fname}}" + path: "{{ fname }}" register: img changed_when: false delegate_to: "{{ download_delegate }}" diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 4986ad25739..57e1bc078c0 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -1,6 +1,10 @@ --- # Set to false to only do certificate management etcd_cluster_setup: true +etcd_events_cluster_setup: false + +# Set to true to separate k8s events to a different etcd cluster +etcd_events_cluster_enabled: false etcd_backup_prefix: "/var/backups" etcd_data_dir: "/var/lib/etcd" @@ -12,22 +16,38 @@ etcd_cert_group: root # Note: This does not set up DNS entries. It simply adds the following DNS # entries to the certificate etcd_cert_alt_names: - - "etcd.{{ system_namespace }}.svc.{{ dns_domain }}" - - "etcd.{{ system_namespace }}.svc" - - "etcd.{{ system_namespace }}" + - "etcd.kube-system.svc.{{ dns_domain }}" + - "etcd.kube-system.svc" + - "etcd.kube-system" - "etcd" +etcd_cert_alt_ips: [] etcd_script_dir: "{{ bin_dir }}/etcd-scripts" etcd_heartbeat_interval: "250" etcd_election_timeout: "5000" +# etcd_snapshot_count: "10000" + +# Parameters for ionice +# -c takes an integer between 0 and 3 or one of the strings none, realtime, best-effort or idle. +# -n takes an integer between 0 (highest priority) and 7 (lowest priority) +# etcd_ionice: "-c2 -n0" + etcd_metrics: "basic" +## A dictionary of extra environment variables to add to etcd.env, formatted like: +## etcd_extra_vars: +## ETCD_VAR1: "value1" +## ETCD_VAR2: "value2" +etcd_extra_vars: {} + # Limits # Limit memory only if <4GB memory on host. 0=unlimited etcd_memory_limit: "{% if ansible_memtotal_mb < 4096 %}512M{% else %}0{% endif %}" +# etcd_quota_backend_bytes: "2G" + # Uncomment to set CPU share for etcd # etcd_cpu_limit: 300m @@ -37,7 +57,7 @@ etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', etcd_compaction_retention: "8" -etcd_vault_mount_path: etcd +etcd_vault_mount_path: "/etcd" # Force clients like etcdctl to use TLS certs (different than peer security) etcd_secure_client: true diff --git a/roles/etcd/files/make-ssl-etcd.sh b/roles/etcd/files/make-ssl-etcd.sh index 5544d6639fb..d661a2a0d05 100755 --- a/roles/etcd/files/make-ssl-etcd.sh +++ b/roles/etcd/files/make-ssl-etcd.sh @@ -65,7 +65,7 @@ if [ -e "$SSLDIR/ca-key.pem" ]; then cp $SSLDIR/{ca.pem,ca-key.pem} . else openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1 - openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1 + openssl req -x509 -new -nodes -key ca-key.pem -days 36500 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1 fi # ETCD member @@ -75,12 +75,12 @@ if [ -n "$MASTERS" ]; then # Member key openssl genrsa -out member-${host}-key.pem 2048 > /dev/null 2>&1 openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${cn}" -config ${CONFIG} > /dev/null 2>&1 - openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days 36500 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 # Admin key openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1 openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=etcd-admin-${cn}" > /dev/null 2>&1 - openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 36500 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 done fi @@ -90,9 +90,14 @@ if [ -n "$HOSTS" ]; then cn="${host%%.*}" openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1 openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=etcd-node-${cn}" > /dev/null 2>&1 - openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 36500 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 done fi # Install certs +if [ -e "$SSLDIR/ca-key.pem" ]; then + # No pass existing CA + rm -f ca.pem ca-key.pem +fi + mv *.pem ${SSLDIR}/ diff --git a/roles/etcd/handlers/backup.yml b/roles/etcd/handlers/backup.yml index 247b2ae004c..d40a3740de0 100644 --- a/roles/etcd/handlers/backup.yml +++ b/roles/etcd/handlers/backup.yml @@ -39,6 +39,8 @@ environment: ETCDCTL_API: 2 retries: 3 + register: backup_v2_command + until: backup_v2_command.rc == 0 delay: "{{ retry_stagger | random + 3 }}" - name: Backup etcd v3 data @@ -48,7 +50,9 @@ snapshot save {{ etcd_backup_directory }}/snapshot.db environment: ETCDCTL_API: 3 - ETCDCTL_CERT: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" retries: 3 + register: etcd_backup_v3_command + until: etcd_backup_v3_command.rc == 0 delay: "{{ retry_stagger | random + 3 }}" diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml index a72cbd515bf..3a46978a68d 100644 --- a/roles/etcd/handlers/main.yml +++ b/roles/etcd/handlers/main.yml @@ -10,7 +10,7 @@ - name: restart etcd-events command: /bin/true notify: - - etcd-events | reload systemd + - etcd | reload systemd - reload etcd-events - wait for etcd-events up @@ -19,9 +19,6 @@ - name: etcd | reload systemd command: systemctl daemon-reload -- name: etcd-events | reload systemd - command: systemctl daemon-reload - - name: reload etcd service: name: etcd diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index d7d3920c65b..674d202e0fd 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -1,20 +1,20 @@ --- -- name: Configure | Check if member is in etcd cluster - shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" - register: etcd_member_in_cluster +- name: Configure | Check if etcd cluster is healthy + shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" + register: etcd_cluster_is_healthy ignore_errors: true changed_when: false check_mode: no - when: is_etcd_master + when: is_etcd_master and etcd_cluster_setup tags: - facts environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" -- name: Configure | Check if member is in etcd-events cluster - shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_access_address }}" - register: etcd_events_member_in_cluster +- name: Configure | Check if etcd-events cluster is healthy + shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'" + register: etcd_events_cluster_is_healthy ignore_errors: true changed_when: false check_mode: no @@ -22,47 +22,112 @@ tags: - facts environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + +- include_tasks: refresh_config.yml + when: is_etcd_master - name: Configure | Copy etcd.service systemd file template: src: "etcd-{{ etcd_deployment_type }}.service.j2" dest: /etc/systemd/system/etcd.service backup: yes - when: is_etcd_master - notify: restart etcd + when: is_etcd_master and etcd_cluster_setup - name: Configure | Copy etcd-events.service systemd file template: - src: "etcd-events-host.service.j2" + src: "etcd-events-{{ etcd_deployment_type }}.service.j2" dest: /etc/systemd/system/etcd-events.service backup: yes - when: is_etcd_master and etcd_deployment_type == "host" and etcd_events_cluster_setup - notify: restart etcd-events + when: is_etcd_master and etcd_events_cluster_setup -- name: Configure | Copy etcd-events.service systemd file - template: - src: "etcd-events-docker.service.j2" - dest: /etc/systemd/system/etcd-events.service - backup: yes - when: is_etcd_master and etcd_deployment_type == "docker" and etcd_events_cluster_setup - notify: restart etcd-events +- name: Configure | reload systemd + command: systemctl daemon-reload + when: is_etcd_master + +- name: Configure | Ensure etcd is running + service: + name: etcd + state: started + enabled: yes + when: is_etcd_master and etcd_cluster_setup + +- name: Configure | Ensure etcd-events is running + service: + name: etcd-events + state: started + enabled: yes + when: is_etcd_master and etcd_events_cluster_setup + +- name: Configure | Check if etcd cluster is healthy + shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" + register: etcd_cluster_is_healthy + until: etcd_cluster_is_healthy.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + ignore_errors: false + changed_when: false + check_mode: no + when: is_etcd_master and etcd_cluster_setup + tags: + - facts + environment: + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + +- name: Configure | Check if etcd-events cluster is healthy + shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'" + register: etcd_events_cluster_is_healthy + until: etcd_events_cluster_is_healthy.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + ignore_errors: false + changed_when: false + check_mode: no + when: is_etcd_master and etcd_events_cluster_setup + tags: + - facts + environment: + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + +- name: Configure | Check if member is in etcd cluster + shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" + register: etcd_member_in_cluster + ignore_errors: true + changed_when: false + check_mode: no + when: is_etcd_master and etcd_cluster_setup + tags: + - facts + environment: + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + +- name: Configure | Check if member is in etcd-events cluster + shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_access_address }}" + register: etcd_events_member_in_cluster + ignore_errors: true + changed_when: false + check_mode: no + when: is_etcd_master and etcd_events_cluster_setup + tags: + - facts + environment: + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - name: Configure | Join member(s) to etcd cluster one at a time include_tasks: join_etcd_member.yml vars: target_node: "{{ item }}" - loop_control: - pause: 10 with_items: "{{ groups['etcd'] }}" - when: inventory_hostname == item and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0 + when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0 - name: Configure | Join member(s) to etcd-events cluster one at a time - include_tasks: join_etcd-evetns_member.yml + include_tasks: join_etcd-events_member.yml vars: target_node: "{{ item }}" - loop_control: - pause: 10 with_items: "{{ groups['etcd'] }}" when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0 diff --git a/roles/etcd/tasks/gen_certs_script.yml b/roles/etcd/tasks/gen_certs_script.yml index 3fbafc52a48..8ef9a3dccec 100644 --- a/roles/etcd/tasks/gen_certs_script.yml +++ b/roles/etcd/tasks/gen_certs_script.yml @@ -15,6 +15,7 @@ owner: root mode: 0700 run_once: yes + when: inventory_hostname == groups['etcd'][0] delegate_to: "{{groups['etcd'][0]}}" - name: "Gen_certs | create etcd cert dir (on {{groups['etcd'][0]}})" @@ -26,6 +27,7 @@ recurse: yes mode: 0700 run_once: yes + when: inventory_hostname == groups['etcd'][0] delegate_to: "{{groups['etcd'][0]}}" - name: Gen_certs | write openssl config @@ -34,7 +36,9 @@ dest: "{{ etcd_config_dir }}/openssl.conf" run_once: yes delegate_to: "{{groups['etcd'][0]}}" - when: gen_certs|default(false) + when: + - gen_certs|default(false) + - inventory_hostname == groups['etcd'][0] - name: Gen_certs | copy certs generation script copy: @@ -43,8 +47,9 @@ mode: 0700 run_once: yes delegate_to: "{{groups['etcd'][0]}}" - when: gen_certs|default(false) - + when: + - gen_certs|default(false) + - inventory_hostname == groups['etcd'][0] - name: Gen_certs | run cert generation script command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" @@ -61,7 +66,9 @@ {% endfor %}" run_once: yes delegate_to: "{{groups['etcd'][0]}}" - when: gen_certs|default(false) + when: + - gen_certs|default(false) + - inventory_hostname == groups['etcd'][0] notify: set etcd_secret_changed - set_fact: @@ -160,5 +167,5 @@ group: "{{ etcd_cert_group }}" state: directory owner: kube - mode: "u=rwX,g-rwx,o-rwx" + mode: "640" recurse: yes diff --git a/roles/etcd/tasks/gen_certs_vault.yml b/roles/etcd/tasks/gen_certs_vault.yml index aa3274bd742..4e3325b4f09 100644 --- a/roles/etcd/tasks/gen_certs_vault.yml +++ b/roles/etcd/tasks/gen_certs_vault.yml @@ -26,6 +26,9 @@ "{{ hostvars[host]['ip'] }}", {%- endif -%} {%- endfor -%} + {%- for cert_alt_ip in etcd_cert_alt_ips -%} + "{{ cert_alt_ip }}", + {%- endfor -%} "127.0.0.1","::1" ] issue_cert_path: "{{ item }}" diff --git a/roles/etcd/tasks/install_docker.yml b/roles/etcd/tasks/install_docker.yml index 58e1485a54a..7859134b013 100644 --- a/roles/etcd/tasks/install_docker.yml +++ b/roles/etcd/tasks/install_docker.yml @@ -9,22 +9,22 @@ retries: 4 delay: "{{ retry_stagger | random + 3 }}" changed_when: false + when: etcd_cluster_setup - name: Install etcd launch script template: src: etcd.j2 dest: "{{ bin_dir }}/etcd" owner: 'root' - mode: 0755 + mode: 0750 backup: yes - notify: restart etcd + when: etcd_cluster_setup - name: Install etcd-events launch script template: src: etcd-events.j2 dest: "{{ bin_dir }}/etcd-events" owner: 'root' - mode: 0755 + mode: 0750 backup: yes when: etcd_events_cluster_setup - notify: restart etcd-events diff --git a/roles/etcd/tasks/install_host.yml b/roles/etcd/tasks/install_host.yml index 9e83905bc89..0dc226e666e 100644 --- a/roles/etcd/tasks/install_host.yml +++ b/roles/etcd/tasks/install_host.yml @@ -1,12 +1,21 @@ --- -- name: Install | Copy etcdctl and etcd binary from docker container - command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy; - {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && - {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcdctl {{ bin_dir }}/etcdctl && - {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcd {{ bin_dir }}/etcd && - {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy" - register: etcd_task_result - until: etcd_task_result.rc == 0 - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" +- name: install | Copy etcd binary from download dir + shell: | + rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcd" "{{ bin_dir }}/etcd" + rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcdctl" "{{ bin_dir }}/etcdctl" changed_when: false + when: etcd_cluster_setup + +- name: install | Set etcd binary permissions + file: + path: "{{ bin_dir }}/etcd" + mode: "0755" + state: file + when: etcd_cluster_setup + +- name: install | Set etcdctl binary permissions + file: + path: "{{ bin_dir }}/etcdctl" + mode: "0755" + state: file + when: etcd_cluster_setup \ No newline at end of file diff --git a/roles/etcd/tasks/install_rkt.yml b/roles/etcd/tasks/install_rkt.yml index 5df623c8b4b..2f693b37115 100644 --- a/roles/etcd/tasks/install_rkt.yml +++ b/roles/etcd/tasks/install_rkt.yml @@ -11,6 +11,7 @@ delay: "{{ retry_stagger | random + 3 }}" changed_when: false environment: "{{proxy_env}}" + when: etcd_cluster_setup - name: Install | Copy etcdctl binary from rkt container command: >- @@ -26,3 +27,4 @@ delay: "{{ retry_stagger | random + 3 }}" changed_when: false environment: "{{proxy_env}}" + when: etcd_cluster_setup diff --git a/roles/etcd/tasks/join_etcd-events_member.yml b/roles/etcd/tasks/join_etcd-events_member.yml index 104ef22dfbf..d5df065f92f 100644 --- a/roles/etcd/tasks/join_etcd-events_member.yml +++ b/roles/etcd/tasks/join_etcd-events_member.yml @@ -1,5 +1,5 @@ --- -- name: Join Member | Add member to cluster +- name: Join Member | Add member to etcd-events cluster shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}" register: member_add_result until: member_add_result.rc == 0 @@ -7,8 +7,8 @@ delay: "{{ retry_stagger | random + 3 }}" when: target_node == inventory_hostname environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - include_tasks: refresh_config.yml vars: @@ -23,17 +23,6 @@ {%- endfor -%} when: target_node == inventory_hostname -- name: Join Member | reload systemd - command: systemctl daemon-reload - when: target_node == inventory_hostname - -- name: Join Member | Ensure etcd-events is running - service: - name: etcd-events - state: started - enabled: yes - when: target_node == inventory_hostname - - name: Join Member | Ensure member is in etcd-events cluster shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_events_access_address }}" register: etcd_events_member_in_cluster @@ -43,5 +32,5 @@ - facts when: target_node == inventory_hostname environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" diff --git a/roles/etcd/tasks/join_etcd_member.yml b/roles/etcd/tasks/join_etcd_member.yml index b7801f0c916..60cfd16cdb6 100644 --- a/roles/etcd/tasks/join_etcd_member.yml +++ b/roles/etcd/tasks/join_etcd_member.yml @@ -1,5 +1,5 @@ --- -- name: Join Member | Add member to cluster +- name: Join Member | Add member to etcd cluster shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}" register: member_add_result until: member_add_result.rc == 0 @@ -7,8 +7,8 @@ delay: "{{ retry_stagger | random + 3 }}" when: target_node == inventory_hostname environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - include_tasks: refresh_config.yml vars: @@ -23,18 +23,7 @@ {%- endfor -%} when: target_node == inventory_hostname -- name: Join Member | reload systemd - command: systemctl daemon-reload - when: target_node == inventory_hostname - -- name: Join Member | Ensure etcd is running - service: - name: etcd - state: started - enabled: yes - when: target_node == inventory_hostname - -- name: Join Member | Ensure member is in cluster +- name: Join Member | Ensure member is in etcd cluster shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" register: etcd_member_in_cluster changed_when: false @@ -43,5 +32,5 @@ - facts when: target_node == inventory_hostname environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" diff --git a/roles/etcd/tasks/join_member.yml b/roles/etcd/tasks/join_member.yml deleted file mode 100644 index b7801f0c916..00000000000 --- a/roles/etcd/tasks/join_member.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -- name: Join Member | Add member to cluster - shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}" - register: member_add_result - until: member_add_result.rc == 0 - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - when: target_node == inventory_hostname - environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" - -- include_tasks: refresh_config.yml - vars: - etcd_peer_addresses: >- - {% for host in groups['etcd'] -%} - {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%} - {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2380, - {%- endif -%} - {%- if loop.last -%} - {{ etcd_member_name }}={{ etcd_peer_url }} - {%- endif -%} - {%- endfor -%} - when: target_node == inventory_hostname - -- name: Join Member | reload systemd - command: systemctl daemon-reload - when: target_node == inventory_hostname - -- name: Join Member | Ensure etcd is running - service: - name: etcd - state: started - enabled: yes - when: target_node == inventory_hostname - -- name: Join Member | Ensure member is in cluster - shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" - register: etcd_member_in_cluster - changed_when: false - check_mode: no - tags: - - facts - when: target_node == inventory_hostname - environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index bb299126be4..db59a983fa7 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -18,58 +18,45 @@ register: "etcd_client_cert_serial_result" changed_when: false when: inventory_hostname in groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort + tags: + - master + - network - name: Set etcd_client_cert_serial set_fact: etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout }}" when: inventory_hostname in groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort + tags: + - master + - network - include_tasks: "install_{{ etcd_deployment_type }}.yml" when: is_etcd_master tags: - upgrade -- import_tasks: set_cluster_health.yml - when: is_etcd_master and etcd_cluster_setup - -- import_tasks: configure.yml - when: is_etcd_master and etcd_cluster_setup +- include_tasks: configure.yml + when: is_etcd_master -- import_tasks: refresh_config.yml - when: is_etcd_master and etcd_cluster_setup +- include_tasks: refresh_config.yml + when: is_etcd_master - name: Restart etcd if certs changed - command: /bin/true - notify: restart etcd - when: is_etcd_master and etcd_secret_changed|default(false) - -- name: Restart etcd-events if certs changed - command: /bin/true - notify: restart etcd - when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed|default(false) - -# reload-systemd -- meta: flush_handlers - -- name: Ensure etcd is running service: name: etcd - state: started + state: restarted enabled: yes - when: is_etcd_master and etcd_cluster_setup + when: is_etcd_master and etcd_cluster_setup and etcd_secret_changed|default(false) -- name: Ensure etcd-events is running +- name: Restart etcd-events if certs changed service: name: etcd-events - state: started + state: restarted enabled: yes - when: is_etcd_master and etcd_events_cluster_setup + when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed|default(false) # After etcd cluster is assembled, make sure that # initial state of the cluster is in `existing` # state insted of `new`. -- import_tasks: set_cluster_health.yml - when: is_etcd_master and etcd_cluster_setup - -- import_tasks: refresh_config.yml - when: is_etcd_master and etcd_cluster_setup +- include_tasks: refresh_config.yml + when: is_etcd_master diff --git a/roles/etcd/tasks/refresh_config.yml b/roles/etcd/tasks/refresh_config.yml index 92766330130..21c308fb018 100644 --- a/roles/etcd/tasks/refresh_config.yml +++ b/roles/etcd/tasks/refresh_config.yml @@ -4,7 +4,7 @@ src: etcd.env.j2 dest: /etc/etcd.env notify: restart etcd - when: is_etcd_master + when: is_etcd_master and etcd_cluster_setup - name: Refresh config | Create etcd-events config file template: diff --git a/roles/etcd/tasks/set_cluster_health.yml b/roles/etcd/tasks/set_cluster_health.yml deleted file mode 100644 index 68e738031a5..00000000000 --- a/roles/etcd/tasks/set_cluster_health.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- name: Configure | Check if etcd cluster is healthy - shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" - register: etcd_cluster_is_healthy - ignore_errors: true - changed_when: false - check_mode: no - when: is_etcd_master - tags: - - facts - environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" - -- name: Configure | Check if etcd-events cluster is healthy - shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'" - register: etcd_events_cluster_is_healthy - ignore_errors: true - changed_when: false - check_mode: no - when: is_etcd_master and etcd_events_cluster_setup - tags: - - facts - environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" diff --git a/roles/etcd/tasks/sync_etcd_master_certs.yml b/roles/etcd/tasks/sync_etcd_master_certs.yml index b810ff775ab..3990e569df3 100644 --- a/roles/etcd/tasks/sync_etcd_master_certs.yml +++ b/roles/etcd/tasks/sync_etcd_master_certs.yml @@ -13,6 +13,8 @@ sync_file: "{{ item }}" sync_file_dir: "{{ etcd_cert_dir }}" sync_file_hosts: [ "{{ inventory_hostname }}" ] + sync_file_owner: kube + sync_file_group: root sync_file_is_cert: true with_items: "{{ etcd_master_cert_list|d([]) }}" diff --git a/roles/etcd/tasks/upd_ca_trust.yml b/roles/etcd/tasks/upd_ca_trust.yml index dd36554fb95..0ff3638601b 100644 --- a/roles/etcd/tasks/upd_ca_trust.yml +++ b/roles/etcd/tasks/upd_ca_trust.yml @@ -8,6 +8,8 @@ /etc/pki/ca-trust/source/anchors/etcd-ca.crt {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%} /etc/ssl/certs/etcd-ca.pem + {%- elif ansible_os_family == "Suse" -%} + /etc/pki/trust/anchors/etcd-ca.pem {%- endif %} tags: - facts @@ -19,9 +21,9 @@ remote_src: true register: etcd_ca_cert -- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS) +- name: Gen_certs | update ca-certificates (Debian/Ubuntu/SUSE/Container Linux by CoreOS) command: update-ca-certificates - when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"] + when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS", "Suse"] - name: Gen_certs | update ca-certificates (RedHat) command: update-ca-trust extract diff --git a/roles/etcd/templates/etcd-events-rkt.service.j2 b/roles/etcd/templates/etcd-events-rkt.service.j2 new file mode 100644 index 00000000000..7886a038b84 --- /dev/null +++ b/roles/etcd/templates/etcd-events-rkt.service.j2 @@ -0,0 +1,31 @@ +[Unit] +Description=etcd events rkt wrapper +Documentation=https://github.com/coreos/etcd +Wants=network.target + +[Service] +Restart=on-failure +RestartSec=10s +TimeoutStartSec=0 +LimitNOFILE=40000 + +ExecStart=/usr/bin/rkt run \ +--uuid-file-save=/var/run/etcd-events.uuid \ +--volume hosts,kind=host,source=/etc/hosts,readOnly=true \ +--mount volume=hosts,target=/etc/hosts \ +--volume=etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ +--mount=volume=etc-ssl-certs,target=/etc/ssl/certs \ +--volume=etcd-cert-dir,kind=host,source={{ etcd_cert_dir }},readOnly=true \ +--mount=volume=etcd-cert-dir,target={{ etcd_cert_dir }} \ +--volume=etcd-data-dir,kind=host,source={{ etcd_events_data_dir }},readOnly=false \ +--mount=volume=etcd-data-dir,target={{ etcd_events_data_dir }} \ +--set-env-file=/etc/etcd-events.env \ +--stage1-from-dir=stage1-fly.aci \ +{{ etcd_image_repo }}:{{ etcd_image_tag }} \ +--name={{ etcd_member_name | default("etcd-events") }} + +ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd-events.uuid +ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/etcd-events.uuid + +[Install] +WantedBy=multi-user.target diff --git a/roles/etcd/templates/etcd-events.env.j2 b/roles/etcd/templates/etcd-events.env.j2 index c168ab03e42..6caeb1cf9da 100644 --- a/roles/etcd/templates/etcd-events.env.j2 +++ b/roles/etcd/templates/etcd-events.env.j2 @@ -1,7 +1,7 @@ ETCD_DATA_DIR={{ etcd_events_data_dir }} ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }} ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }} -ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %} +ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %} ETCD_METRICS={{ etcd_metrics }} ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2381,https://127.0.0.1:2381 @@ -13,6 +13,12 @@ ETCD_NAME={{ etcd_member_name }}-events ETCD_PROXY=off ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }} ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }} +{% if etcd_snapshot_count is defined %} +ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }} +{% endif %} +{% if etcd_quota_backend_bytes is defined %} +ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} +{% endif %} # TLS settings ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem @@ -24,3 +30,7 @@ ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }} + +{% if host_architecture != "amd64" -%} +ETCD_UNSUPPORTED_ARCH={{host_architecture}} +{%- endif %} diff --git a/roles/etcd/templates/etcd.env.j2 b/roles/etcd/templates/etcd.env.j2 index 6a917d127a3..a682f1f71e0 100644 --- a/roles/etcd/templates/etcd.env.j2 +++ b/roles/etcd/templates/etcd.env.j2 @@ -1,3 +1,4 @@ +# Environment file for etcd {{ etcd_version }} ETCD_DATA_DIR={{ etcd_data_dir }} ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }} ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }} @@ -13,6 +14,12 @@ ETCD_NAME={{ etcd_member_name }} ETCD_PROXY=off ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }} +{% if etcd_snapshot_count is defined %} +ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }} +{% endif %} +{% if etcd_quota_backend_bytes is defined %} +ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} +{% endif %} # TLS settings ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem @@ -24,3 +31,11 @@ ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }} + +{% for key, value in etcd_extra_vars.items() %} +{{ key }}={{ value }} +{% endfor %} + +{% if host_architecture != "amd64" -%} +ETCD_UNSUPPORTED_ARCH={{host_architecture}} +{%- endif %} diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 old mode 100644 new mode 100755 index 9ac08e0738a..a6628d8fb97 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -6,16 +6,19 @@ -v /etc/ssl/certs:/etc/ssl/certs:ro \ -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \ -v {{ etcd_data_dir }}:{{ etcd_data_dir }}:rw \ - {% if etcd_memory_limit is defined %} +{% if etcd_memory_limit is defined %} --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \ - {% endif %} - {% if etcd_cpu_limit is defined %} +{% endif %} +{% if etcd_cpu_limit is defined %} --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \ - {% endif %} - {% if etcd_blkio_weight is defined %} +{% endif %} +{% if etcd_blkio_weight is defined %} --blkio-weight={{ etcd_blkio_weight }} \ - {% endif %} +{% endif %} --name={{ etcd_member_name | default("etcd") }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ +{% if etcd_ionice is defined %} + /bin/ionice {{ etcd_ionice }} \ +{% endif %} /usr/local/bin/etcd \ "$@" diff --git a/roles/etcd/templates/openssl.conf.j2 b/roles/etcd/templates/openssl.conf.j2 index 48327f0bfa6..402417827c1 100644 --- a/roles/etcd/templates/openssl.conf.j2 +++ b/roles/etcd/templates/openssl.conf.j2 @@ -1,4 +1,4 @@ -[req] +{% set counter = {'dns': 2,'ip': 1,} %}{% macro increment(dct, key, inc=1)%}{% if dct.update({key: dct[key] + inc}) %} {% endif %}{% endmacro %}[req] req_extensions = v3_req distinguished_name = req_distinguished_name @@ -25,19 +25,21 @@ authorityKeyIdentifier=keyid:always,issuer [alt_names] DNS.1 = localhost {% for host in groups['etcd'] %} -DNS.{{ 1 + loop.index }} = {{ host }} +DNS.{{ counter["dns"] }} = {{ host }}{{ increment(counter, 'dns') }} {% endfor %} -{% if loadbalancer_apiserver is defined %} -{% set idx = groups['etcd'] | length | int + 2 %} -DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }} +{% if apiserver_loadbalancer_domain_name is defined %} +DNS.{{ counter["dns"] }} = {{ apiserver_loadbalancer_domain_name }}{{ increment(counter, 'dns') }} {% endif %} -{% set idx = groups['etcd'] | length | int + 3 %} {% for etcd_alt_name in etcd_cert_alt_names %} -DNS.{{ idx + 1 + loop.index }} = {{ etcd_alt_name }} +DNS.{{ counter["dns"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }} {% endfor %} {% for host in groups['etcd'] %} -IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} -IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +{% if hostvars[host]['access_ip'] is defined %} +IP.{{ counter["ip"] }} = {{ hostvars[host]['access_ip'] }}{{ increment(counter, 'ip') }} +{% endif %} +IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}{{ increment(counter, 'ip') }} +{% endfor %} +{% for cert_alt_ip in etcd_cert_alt_ips %} +IP.{{ counter["ip"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }} {% endfor %} -{% set idx = groups['etcd'] | length | int * 2 + 1 %} -IP.{{ idx }} = 127.0.0.1 +IP.{{ counter["ip"] }} = 127.0.0.1 diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 350f663a132..6a60e09d84e 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -1,6 +1,6 @@ --- # Versions -kubedns_version: 1.14.8 +kubedns_version: 1.14.10 kubednsautoscaler_version: 1.1.2 # Limits for dnsmasq/kubedns apps @@ -10,14 +10,19 @@ dns_memory_requests: 70Mi kubedns_min_replicas: 2 kubedns_nodes_per_replica: 10 +# CoreDNS +coredns_replicas: 2 + # Images -kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-amd64" +image_arch: "{{host_architecture}}" + +kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-{{ image_arch }}" kubedns_image_tag: "{{ kubedns_version }}" -dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64" +dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-{{ image_arch }}" dnsmasq_nanny_image_tag: "{{ kubedns_version }}" -dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-amd64" +dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-{{ image_arch }}" dnsmasq_sidecar_image_tag: "{{ kubedns_version }}" -kubednsautoscaler_image_repo: "gcr.io/google_containers/cluster-proportional-autoscaler-amd64" +kubednsautoscaler_image_repo: "gcr.io/google_containers/cluster-proportional-autoscaler-{{ image_arch }}" kubednsautoscaler_image_tag: "{{ kubednsautoscaler_version }}" # Netchecker @@ -38,10 +43,16 @@ netchecker_server_memory_limit: 256M netchecker_server_cpu_requests: 50m netchecker_server_memory_requests: 64M +# SecurityContext when PodSecurityPolicy is enabled +netchecker_agent_user: 1000 +netchecker_server_user: 1000 +netchecker_agent_group: 1000 +netchecker_server_group: 1000 + # Dashboard dashboard_enabled: true -dashboard_image_repo: gcr.io/google_containers/kubernetes-dashboard-amd64 -dashboard_image_tag: v1.8.3 +dashboard_image_repo: gcr.io/google_containers/kubernetes-dashboard-{{ image_arch }} +dashboard_image_tag: v1.10.0 # Limits for dashboard dashboard_cpu_limit: 100m @@ -56,11 +67,9 @@ dashboard_certs_secret_name: kubernetes-dashboard-certs dashboard_tls_key_file: dashboard.key dashboard_tls_cert_file: dashboard.crt +# Override dashboard default settings +dashboard_token_ttl: 900 + # SSL etcd_cert_dir: "/etc/ssl/etcd/ssl" canal_cert_dir: "/etc/canal/certs" - -rbac_resources: - - sa - - clusterrole - - clusterrolebinding diff --git a/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml b/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml new file mode 100644 index 00000000000..e77f1e7991e --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml @@ -0,0 +1,54 @@ +--- +- name: Kubernetes Apps | Delete old CoreDNS resources + kube: + name: "coredns" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item }}" + state: absent + with_items: + - 'deploy' + - 'configmap' + - 'svc' + tags: + - upgrade + +- name: Kubernetes Apps | Delete kubeadm CoreDNS + kube: + name: "coredns" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "deploy" + state: absent + when: + - kubeadm_enabled|default(false) + - kubeadm_init.changed|default(false) + - inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Apps | Delete old KubeDNS resources + kube: + name: "kube-dns" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item }}" + state: absent + with_items: + - 'deploy' + - 'svc' + tags: + - upgrade + +- name: Kubernetes Apps | Delete kubeadm KubeDNS + kube: + name: "kube-dns" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item }}" + state: absent + with_items: + - 'deploy' + - 'svc' + when: + - kubeadm_enabled|default(false) + - kubeadm_init.changed|default(false) + - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ansible/tasks/coredns.yml b/roles/kubernetes-apps/ansible/tasks/coredns.yml new file mode 100644 index 00000000000..c52cf7ba8fc --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/coredns.yml @@ -0,0 +1,37 @@ +--- +- name: Kubernetes Apps | Lay Down CoreDNS Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - { name: coredns, file: coredns-config.yml, type: configmap } + - { name: coredns, file: coredns-sa.yml, type: sa } + - { name: coredns, file: coredns-deployment.yml, type: deployment } + - { name: coredns, file: coredns-svc.yml, type: svc } + - { name: coredns, file: coredns-clusterrole.yml, type: clusterrole } + - { name: coredns, file: coredns-clusterrolebinding.yml, type: clusterrolebinding } + register: coredns_manifests + vars: + clusterIP: "{{ skydns_server }}" + when: + - dns_mode in ['coredns', 'coredns_dual'] + - inventory_hostname == groups['kube-master'][0] + tags: + - coredns + +- name: Kubernetes Apps | Lay Down Secondary CoreDNS Template + template: + src: "{{ item.src }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - { name: coredns, src: coredns-deployment.yml, file: coredns-deployment-secondary.yml, type: deployment } + - { name: coredns, src: coredns-svc.yml, file: coredns-svc-secondary.yml, type: svc } + register: coredns_secondary_manifests + vars: + clusterIP: "{{ skydns_server_secondary }}" + coredns_ordinal_suffix: "-secondary" + when: + - dns_mode == 'coredns_dual' + - inventory_hostname == groups['kube-master'][0] + tags: + - coredns diff --git a/roles/kubernetes-apps/ansible/tasks/dashboard.yml b/roles/kubernetes-apps/ansible/tasks/dashboard.yml index ce56bd5d10a..4c9ad5c7426 100644 --- a/roles/kubernetes-apps/ansible/tasks/dashboard.yml +++ b/roles/kubernetes-apps/ansible/tasks/dashboard.yml @@ -22,7 +22,7 @@ - name: Kubernetes Apps | Start dashboard kube: name: "{{ item.item.name }}" - namespace: "{{ system_namespace }}" + namespace: "kube-system" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" diff --git a/roles/kubernetes-apps/ansible/tasks/kubedns.yml b/roles/kubernetes-apps/ansible/tasks/kubedns.yml new file mode 100644 index 00000000000..e7bf8298fa2 --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/kubedns.yml @@ -0,0 +1,42 @@ +--- + +- name: Kubernetes Apps | Lay Down KubeDNS Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - { name: kube-dns, file: kubedns-sa.yml, type: sa } + - { name: kube-dns, file: kubedns-deploy.yml, type: deployment } + - { name: kube-dns, file: kubedns-svc.yml, type: svc } + - { name: kubedns-autoscaler, file: kubedns-autoscaler-sa.yml, type: sa } + - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrole.yml, type: clusterrole } + - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding } + - { name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment } + register: kubedns_manifests + when: + - dns_mode in ['kubedns','dnsmasq_kubedns'] + - inventory_hostname == groups['kube-master'][0] + tags: + - dnsmasq + - kubedns + +# see https://github.com/kubernetes/kubernetes/issues/45084, only needed for "old" kube-dns +- name: Kubernetes Apps | Patch system:kube-dns ClusterRole + command: > + {{ bin_dir }}/kubectl patch clusterrole system:kube-dns + --patch='{ + "rules": [ + { + "apiGroups" : [""], + "resources" : ["endpoints", "services"], + "verbs": ["list", "watch", "get"] + } + ] + }' + when: + - dns_mode in ['kubedns', 'dnsmasq_kubedns'] + - inventory_hostname == groups['kube-master'][0] + - rbac_enabled and kubedns_version|version_compare("1.11.0", "<", strict=True) + tags: + - dnsmasq + - kubedns diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index a25d595ebb6..62169d27d49 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -11,84 +11,56 @@ delay: 2 when: inventory_hostname == groups['kube-master'][0] -- name: Kubernetes Apps | Delete old kubedns resources - kube: - name: "kubedns" - namespace: "{{ system_namespace }}" - kubectl: "{{ bin_dir }}/kubectl" - resource: "{{ item }}" - state: absent - with_items: - - 'deploy' - - 'svc' +- name: Kubernetes Apps | Cleanup DNS + import_tasks: tasks/cleanup_dns.yml + when: + - inventory_hostname == groups['kube-master'][0] tags: - upgrade + - dnsmasq + - coredns + - kubedns -- name: Kubernetes Apps | Delete kubeadm kubedns - kube: - name: "kubedns" - namespace: "{{ system_namespace }}" - kubectl: "{{ bin_dir }}/kubectl" - resource: "deploy" - state: absent +- name: Kubernetes Apps | CoreDNS + import_tasks: "tasks/coredns.yml" when: - - kubeadm_enabled|default(false) - - kubeadm_init.changed|default(false) + - dns_mode in ['coredns', 'coredns_dual'] - inventory_hostname == groups['kube-master'][0] - -- name: Kubernetes Apps | Lay Down KubeDNS Template - template: - src: "{{ item.file }}.j2" - dest: "{{ kube_config_dir }}/{{ item.file }}" - with_items: - - { name: kube-dns, file: kubedns-sa.yml, type: sa } - - { name: kube-dns, file: kubedns-deploy.yml, type: deployment } - - { name: kube-dns, file: kubedns-svc.yml, type: svc } - - { name: kubedns-autoscaler, file: kubedns-autoscaler-sa.yml, type: sa } - - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrole.yml, type: clusterrole } - - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding } - - { name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment } - register: manifests - when: - - dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] - - rbac_enabled or item.type not in rbac_resources tags: - - dnsmasq + - coredns -# see https://github.com/kubernetes/kubernetes/issues/45084, only needed for "old" kube-dns -- name: Kubernetes Apps | Patch system:kube-dns ClusterRole - command: > - {{ bin_dir }}/kubectl patch clusterrole system:kube-dns - --patch='{ - "rules": [ - { - "apiGroups" : [""], - "resources" : ["endpoints", "services"], - "verbs": ["list", "watch", "get"] - } - ] - }' +- name: Kubernetes Apps | KubeDNS + import_tasks: "tasks/kubedns.yml" when: - - dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] - - rbac_enabled and kubedns_version|version_compare("1.11.0", "<", strict=True) + - dns_mode in ['kubedns', 'dnsmasq_kubedns'] + - inventory_hostname == groups['kube-master'][0] tags: - dnsmasq - name: Kubernetes Apps | Start Resources kube: name: "{{ item.item.name }}" - namespace: "{{ system_namespace }}" + namespace: "kube-system" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" - with_items: "{{ manifests.results }}" + with_items: + - "{{ kubedns_manifests.results | default({}) }}" + - "{{ coredns_manifests.results | default({}) }}" + - "{{ coredns_secondary_manifests.results | default({}) }}" when: - dns_mode != 'none' - inventory_hostname == groups['kube-master'][0] - not item|skipped + register: resource_result + until: resource_result|succeeded + retries: 4 + delay: 5 tags: - dnsmasq + - coredns + - kubedns - name: Kubernetes Apps | Netchecker import_tasks: tasks/netchecker.yml diff --git a/roles/kubernetes-apps/ansible/tasks/netchecker.yml b/roles/kubernetes-apps/ansible/tasks/netchecker.yml index c07f07bba37..655ef744b0f 100644 --- a/roles/kubernetes-apps/ansible/tasks/netchecker.yml +++ b/roles/kubernetes-apps/ansible/tasks/netchecker.yml @@ -2,7 +2,7 @@ - name: Kubernetes Apps | Check if netchecker-server manifest already exists stat: - path: "{{ kube_config_dir }}/netchecker-server-deployment.yml.j2" + path: "{{ kube_config_dir }}/netchecker-server-deployment.yml" register: netchecker_server_manifest tags: - facts @@ -20,22 +20,35 @@ tags: - upgrade +- name: Kubernetes Apps | Netchecker Templates list + set_fact: + netchecker_templates: + - {file: netchecker-agent-sa.yml, type: sa, name: netchecker-agent} + - {file: netchecker-agent-ds.yml, type: ds, name: netchecker-agent} + - {file: netchecker-agent-hostnet-ds.yml, type: ds, name: netchecker-agent-hostnet} + - {file: netchecker-server-sa.yml, type: sa, name: netchecker-server} + - {file: netchecker-server-clusterrole.yml, type: clusterrole, name: netchecker-server} + - {file: netchecker-server-clusterrolebinding.yml, type: clusterrolebinding, name: netchecker-server} + - {file: netchecker-server-deployment.yml, type: deployment, name: netchecker-server} + - {file: netchecker-server-svc.yml, type: svc, name: netchecker-service} + netchecker_templates_for_psp: + - {file: netchecker-agent-hostnet-psp.yml, type: podsecuritypolicy, name: netchecker-agent-hostnet-policy} + - {file: netchecker-agent-hostnet-clusterrole.yml, type: clusterrole, name: netchecker-agent} + - {file: netchecker-agent-hostnet-clusterrolebinding.yml, type: clusterrolebinding, name: netchecker-agent} + +- name: Kubernetes Apps | Append extra templates to Netchecker Templates list for PodSecurityPolicy + set_fact: + netchecker_templates: "{{ netchecker_templates_for_psp + netchecker_templates}}" + when: podsecuritypolicy_enabled + - name: Kubernetes Apps | Lay Down Netchecker Template template: - src: "{{item.file}}" + src: "{{item.file}}.j2" dest: "{{kube_config_dir}}/{{item.file}}" - with_items: - - {file: netchecker-agent-ds.yml.j2, type: ds, name: netchecker-agent} - - {file: netchecker-agent-hostnet-ds.yml.j2, type: ds, name: netchecker-agent-hostnet} - - {file: netchecker-server-sa.yml.j2, type: sa, name: netchecker-server} - - {file: netchecker-server-clusterrole.yml.j2, type: clusterrole, name: netchecker-server} - - {file: netchecker-server-clusterrolebinding.yml.j2, type: clusterrolebinding, name: netchecker-server} - - {file: netchecker-server-deployment.yml.j2, type: deployment, name: netchecker-server} - - {file: netchecker-server-svc.yml.j2, type: svc, name: netchecker-service} + with_items: "{{ netchecker_templates }}" register: manifests when: - inventory_hostname == groups['kube-master'][0] - - rbac_enabled or item.type not in rbac_resources - name: Kubernetes Apps | Purge old Netchecker server kube: diff --git a/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 new file mode 100644 index 00000000000..4136d603e98 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 @@ -0,0 +1,19 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: Reconcile + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch diff --git a/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 new file mode 100644 index 00000000000..89becd5b44b --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: EnsureExists + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system diff --git a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 new file mode 100644 index 00000000000..34cd4b77e4d --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | + .:53 { + errors + health + kubernetes {{ dns_domain }} in-addr.arpa ip6.arpa { + pods insecure + upstream /etc/resolv.conf + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + proxy . /etc/resolv.conf + cache 30 + } diff --git a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 new file mode 100644 index 00000000000..27c0576a1f7 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 @@ -0,0 +1,95 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: coredns{{ coredns_ordinal_suffix | default('') }} + namespace: kube-system + labels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" +spec: + replicas: {{ coredns_replicas }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 10% + selector: + matchLabels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + template: + metadata: + labels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: coredns + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Exists" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" + labelSelector: + matchLabels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "true" + containers: + - name: coredns + image: "{{ coredns_image_repo }}:{{ coredns_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: {{ dns_memory_limit }} + requests: + cpu: {{ dns_cpu_requests }} + memory: {{ dns_memory_requests }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile diff --git a/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 new file mode 100644 index 00000000000..64d9c4dae27 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 new file mode 100644 index 00000000000..193de10eb97 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns{{ coredns_ordinal_suffix | default('') }} + namespace: kube-system + labels: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} + clusterIP: {{ clusterIP }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 index b1ba1481de2..41f6716e7ff 100644 --- a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 @@ -25,7 +25,7 @@ metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard-certs - namespace: {{ system_namespace }} + namespace: kube-system type: Opaque --- @@ -37,7 +37,7 @@ metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard - namespace: {{ system_namespace }} + namespace: kube-system --- # ------------------- Dashboard Role & Role Binding ------------------- # @@ -46,7 +46,7 @@ kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kubernetes-dashboard-minimal - namespace: {{ system_namespace }} + namespace: kube-system rules: # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret. - apiGroups: [""] @@ -81,7 +81,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: kubernetes-dashboard-minimal - namespace: {{ system_namespace }} + namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -89,7 +89,7 @@ roleRef: subjects: - kind: ServiceAccount name: kubernetes-dashboard - namespace: {{ system_namespace }} + namespace: kube-system --- # ------------------- Gross Hack For anonymous auth through api proxy ------------------- # @@ -103,7 +103,7 @@ rules: resources: ["services/proxy"] resourceNames: ["https:kubernetes-dashboard:"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -- nonResourceURLs: ["/ui", "/ui/*", "/api/v1/namespaces/{{ system_namespace }}/services/https:kubernetes-dashboard:/proxy/*"] +- nonResourceURLs: ["/ui", "/ui/*", "/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/*"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] --- @@ -128,7 +128,7 @@ metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard - namespace: {{ system_namespace }} + namespace: kube-system spec: replicas: 1 revisionHistoryLimit: 10 @@ -166,6 +166,7 @@ spec: # If not specified, Dashboard will attempt to auto discover the API server and connect # to it. Uncomment only if the default does not work. # - --apiserver-host=http://my-address:port + - --token-ttl={{ dashboard_token_ttl }} volumeMounts: - name: kubernetes-dashboard-certs mountPath: /certs @@ -199,8 +200,9 @@ apiVersion: v1 metadata: labels: k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" name: kubernetes-dashboard - namespace: {{ system_namespace }} + namespace: kube-system spec: ports: - port: 443 diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrole.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrole.yml.j2 index f80d3d90c17..e29ed4dac54 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrole.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrole.yml.j2 @@ -17,7 +17,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: cluster-proportional-autoscaler - namespace: {{ system_namespace }} + namespace: kube-system rules: - apiGroups: [""] resources: ["nodes"] diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrolebinding.yml.j2 index eb76f2d4ecb..3b11c6b9fcb 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrolebinding.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrolebinding.yml.j2 @@ -17,11 +17,11 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: cluster-proportional-autoscaler - namespace: {{ system_namespace }} + namespace: kube-system subjects: - kind: ServiceAccount name: cluster-proportional-autoscaler - namespace: {{ system_namespace }} + namespace: kube-system roleRef: kind: ClusterRole name: cluster-proportional-autoscaler diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-sa.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-sa.yml.j2 index 542ae86cec4..4c440f653f1 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-sa.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-sa.yml.j2 @@ -17,4 +17,4 @@ kind: ServiceAccount apiVersion: v1 metadata: name: cluster-proportional-autoscaler - namespace: {{ system_namespace }} + namespace: kube-system diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 index df92ee6156b..e726e8d2a7f 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2 @@ -17,7 +17,7 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: kubedns-autoscaler - namespace: {{ system_namespace }} + namespace: kube-system labels: k8s-app: kubedns-autoscaler kubernetes.io/cluster-service: "true" @@ -28,9 +28,29 @@ spec: labels: k8s-app: kubedns-autoscaler spec: + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux tolerations: - effect: NoSchedule - operator: Exists + operator: Equal + key: node-role.kubernetes.io/master + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" + labelSelector: + matchLabels: + k8s-app: kubedns-autoscaler + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "true" containers: - name: autoscaler image: "{{ kubednsautoscaler_image_repo }}:{{ kubednsautoscaler_image_tag }}" @@ -40,13 +60,11 @@ spec: memory: "10Mi" command: - /cluster-proportional-autoscaler - - --namespace={{ system_namespace }} + - --namespace=kube-system - --configmap=kubedns-autoscaler # Should keep target in sync with cluster/addons/dns/kubedns-controller.yaml.base - --target=Deployment/kube-dns - --default-params={"linear":{"nodesPerReplica":{{ kubedns_nodes_per_replica }},"min":{{ kubedns_min_replicas }}}} - --logtostderr=true - --v=2 -{% if rbac_enabled %} serviceAccountName: cluster-proportional-autoscaler -{% endif %} diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 index 682bdf49130..96ef72283ef 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 @@ -3,7 +3,7 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: kube-dns - namespace: "{{system_namespace}}" + namespace: kube-system labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" @@ -27,11 +27,31 @@ spec: annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux tolerations: - key: "CriticalAddonsOnly" operator: "Exists" - - effect: NoSchedule - operator: Exists + - effect: "NoSchedule" + operator: "Equal" + key: "node-role.kubernetes.io/master" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" + labelSelector: + matchLabels: + k8s-app: kube-dns + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "true" volumes: - name: kube-dns-config configMap: @@ -110,6 +130,7 @@ spec: - -- - -k - --cache-size=1000 + - --dns-loop-detect - --log-facility=- - --server=/{{ dns_domain }}/127.0.0.1#10053 - --server=/in-addr.arpa/127.0.0.1#10053 @@ -154,6 +175,4 @@ spec: memory: 20Mi cpu: 10m dnsPolicy: Default # Don't use cluster DNS. -{% if rbac_enabled %} serviceAccountName: kube-dns -{% endif %} diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-sa.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-sa.yml.j2 index f399fd6f4b4..296a3a93820 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-sa.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-sa.yml.j2 @@ -3,6 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: kube-dns - namespace: {{ system_namespace }} + namespace: kube-system labels: kubernetes.io/cluster-service: "true" diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-svc.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-svc.yml.j2 index 1c4710db13b..6bc5f9240e1 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-svc.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-svc.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Service metadata: name: kube-dns - namespace: {{ system_namespace }} + namespace: kube-system labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 index 4f32214ebd9..a2c4850c442 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 @@ -15,6 +15,9 @@ spec: tolerations: - effect: NoSchedule operator: Exists + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: netchecker-agent image: "{{ agent_img }}" @@ -40,6 +43,10 @@ spec: requests: cpu: {{ netchecker_agent_cpu_requests }} memory: {{ netchecker_agent_memory_requests }} + securityContext: + runAsUser: {{ netchecker_agent_user | default('0') }} + runAsGroup: {{ netchecker_agent_group | default('0') }} + serviceAccountName: netchecker-agent updateStrategy: rollingUpdate: maxUnavailable: 100% diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-clusterrole.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-clusterrole.yml.j2 new file mode 100644 index 00000000000..f364b428388 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-clusterrole.yml.j2 @@ -0,0 +1,14 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: psp:netchecker-agent-hostnet + namespace: {{ netcheck_namespace }} +rules: + - apiGroups: + - policy + resourceNames: + - netchecker-agent-hostnet + resources: + - podsecuritypolicies + verbs: + - use diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-clusterrolebinding.yml.j2 new file mode 100644 index 00000000000..766faa28cb5 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-clusterrolebinding.yml.j2 @@ -0,0 +1,13 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: psp:netchecker-agent-hostnet + namespace: {{ netcheck_namespace }} +subjects: + - kind: ServiceAccount + name: netchecker-agent-hostnet + namespace: {{ netcheck_namespace }} +roleRef: + kind: ClusterRole + name: psp:netchecker-agent-hostnet + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 index 76fca481283..f046e8f4b58 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 @@ -13,6 +13,9 @@ spec: app: netchecker-agent-hostnet spec: hostNetwork: True + # When having win nodes in cluster without this patch, this pod cloud try to be created in windows + nodeSelector: + beta.kubernetes.io/os: linux {% if kube_version | version_compare('v1.6', '>=') %} dnsPolicy: ClusterFirstWithHostNet {% endif %} @@ -44,6 +47,10 @@ spec: requests: cpu: {{ netchecker_agent_cpu_requests }} memory: {{ netchecker_agent_memory_requests }} + securityContext: + runAsUser: {{ netchecker_agent_user | default('0') }} + runAsGroup: {{ netchecker_agent_group | default('0') }} + serviceAccountName: netchecker-agent updateStrategy: rollingUpdate: maxUnavailable: 100% diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-psp.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-psp.yml.j2 new file mode 100644 index 00000000000..32fb0c1a0e1 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-psp.yml.j2 @@ -0,0 +1,45 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: netchecker-agent-hostnet + annotations: + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' +{% if apparmor_enabled %} + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' +{% endif %} + labels: + kubernetes.io/cluster-service: 'true' + addonmanager.kubernetes.io/mode: Reconcile +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2 new file mode 100644 index 00000000000..d842faa6c07 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2 @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: netchecker-agent + namespace: {{ netcheck_namespace }} + labels: + kubernetes.io/cluster-service: "true" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2 index 7a8c1d2731d..19bdc8b1fa1 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2 @@ -7,3 +7,6 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["list"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ['*'] diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 index 6e2738e6fc7..1a858683d2a 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 @@ -23,6 +23,9 @@ spec: requests: cpu: {{ netchecker_server_cpu_requests }} memory: {{ netchecker_server_memory_requests }} + securityContext: + runAsUser: {{ netchecker_server_user | default('0') }} + runAsGroup: {{ netchecker_server_group | default('0') }} ports: - containerPort: 8081 args: @@ -33,6 +36,4 @@ spec: tolerations: - effect: NoSchedule operator: Exists -{% if rbac_enabled %} serviceAccountName: netchecker-server -{% endif %} diff --git a/roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml b/roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml deleted file mode 100644 index 9a3bca1ef81..00000000000 --- a/roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -cephfs_provisioner_image_repo: quay.io/kubespray/cephfs-provisioner -cephfs_provisioner_image_tag: 92295a30 - -cephfs_provisioner_namespace: "{{ system_namespace }}" -cephfs_provisioner_cluster: ceph -cephfs_provisioner_monitors: [] -cephfs_provisioner_admin_id: admin -cephfs_provisioner_secret: secret -cephfs_provisioner_storage_class: cephfs diff --git a/roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml b/roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml deleted file mode 100644 index 6e854f05ea7..00000000000 --- a/roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- - -- name: CephFS Provisioner | Create addon dir - file: - path: "{{ kube_config_dir }}/addons/cephfs_provisioner" - owner: root - group: root - mode: 0755 - recurse: true - -- name: CephFS Provisioner | Create manifests - template: - src: "{{ item.file }}.j2" - dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}" - with_items: - - { name: cephfs-provisioner-sa, file: cephfs-provisioner-sa.yml, type: sa } - - { name: cephfs-provisioner-role, file: cephfs-provisioner-role.yml, type: role } - - { name: cephfs-provisioner-rolebinding, file: cephfs-provisioner-rolebinding.yml, type: rolebinding } - - { name: cephfs-provisioner-clusterrole, file: cephfs-provisioner-clusterrole.yml, type: clusterrole } - - { name: cephfs-provisioner-clusterrolebinding, file: cephfs-provisioner-clusterrolebinding.yml, type: clusterrolebinding } - - { name: cephfs-provisioner-deploy, file: cephfs-provisioner-deploy.yml, type: deploy } - - { name: cephfs-provisioner-secret, file: cephfs-provisioner-secret.yml, type: secret } - - { name: cephfs-provisioner-sc, file: cephfs-provisioner-sc.yml, type: sc } - register: cephfs_manifests - when: inventory_hostname == groups['kube-master'][0] - -- name: CephFS Provisioner | Apply manifests - kube: - name: "{{ item.item.name }}" - namespace: "{{ system_namespace }}" - kubectl: "{{ bin_dir }}/kubectl" - resource: "{{ item.item.type }}" - filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}" - state: "latest" - with_items: "{{ cephfs_manifests.results }}" - when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/cloud_controller/oci/defaults/main.yml b/roles/kubernetes-apps/cloud_controller/oci/defaults/main.yml new file mode 100644 index 00000000000..ccfb7007712 --- /dev/null +++ b/roles/kubernetes-apps/cloud_controller/oci/defaults/main.yml @@ -0,0 +1,5 @@ +--- + +oci_security_list_management: All +oci_use_instance_principals: false +oci_cloud_controller_version: 0.5.0 diff --git a/roles/kubernetes-apps/cloud_controller/oci/tasks/credentials-check.yml b/roles/kubernetes-apps/cloud_controller/oci/tasks/credentials-check.yml new file mode 100644 index 00000000000..b6098686f96 --- /dev/null +++ b/roles/kubernetes-apps/cloud_controller/oci/tasks/credentials-check.yml @@ -0,0 +1,56 @@ +--- + +- name: "OCI Cloud Controller | Credentials Check | oci_private_key" + fail: + msg: "oci_private_key is missing" + when: (oci_use_instance_principals == false) and + (oci_private_key is not defined or oci_private_key == "") + +- name: "OCI Cloud Controller | Credentials Check | oci_region_id" + fail: + msg: "oci_region_id is missing" + when: (oci_use_instance_principals == false) and + (oci_region_id is not defined or oci_region_id == "") + +- name: "OCI Cloud Controller | Credentials Check | oci_tenancy_id" + fail: + msg: "oci_tenancy_id is missing" + when: (oci_use_instance_principals == false) and + (oci_tenancy_id is not defined or oci_tenancy_id == "") + +- name: "OCI Cloud Controller | Credentials Check | oci_user_id" + fail: + msg: "oci_user_id is missing" + when: (oci_use_instance_principals == false) and + (oci_user_id is not defined or oci_user_id == "") + +- name: "OCI Cloud Controller | Credentials Check | oci_user_fingerprint" + fail: + msg: "oci_user_fingerprint is missing" + when: (oci_use_instance_principals == false) and + (oci_user_fingerprint is not defined or oci_user_fingerprint == "") + +- name: "OCI Cloud Controller | Credentials Check | oci_compartment_id" + fail: + msg: "oci_compartment_id is missing. This is the compartment in which the cluster resides" + when: oci_compartment_id is not defined or oci_compartment_id == "" + +- name: "OCI Cloud Controller | Credentials Check | oci_vnc_id" + fail: + msg: "oci_vnc_id is missin. This is the Virtual Cloud Network in which the cluster resides" + when: oci_vnc_id is not defined or oci_vnc_id == "" + +- name: "OCI Cloud Controller | Credentials Check | oci_subnet1_id" + fail: + msg: "oci_subnet1_id is missing. This is the first subnet to which loadbalancers will be added" + when: oci_subnet1_id is not defined or oci_subnet1_id == "" + +- name: "OCI Cloud Controller | Credentials Check | oci_subnet2_id" + fail: + msg: "oci_subnet2_id is missing. Two subnets are required for load balancer high availability" + when: oci_subnet2_id is not defined or oci_subnet2_id == "" + +- name: "OCI Cloud Controller | Credentials Check | oci_security_list_management" + fail: + msg: "oci_security_list_management is missing, or not defined correctly. Valid options are (All, Frontend, None)." + when: oci_security_list_management is not defined or oci_security_list_management not in ["All", "Frontend", "None"] \ No newline at end of file diff --git a/roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml b/roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml new file mode 100644 index 00000000000..37e5962d3dc --- /dev/null +++ b/roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml @@ -0,0 +1,51 @@ +--- + +- include: credentials-check.yml + tags: oci + +- name: "OCI Cloud Controller | Generate Configuration" + template: + src: controller-manager-config.yml.j2 + dest: /tmp/controller-manager-config.yml + register: controller_manager_config + when: inventory_hostname == groups['kube-master'][0] + tags: oci + +- name: "OCI Cloud Controller | Encode Configuration" + set_fact: + controller_manager_config_base64: "{{ lookup('file', '/tmp/controller-manager-config.yml') | b64encode }}" + when: inventory_hostname == groups['kube-master'][0] + tags: oci + +- name: "OCI Cloud Controller | Apply Configuration To Secret" + template: + src: cloud-provider.yml.j2 + dest: /tmp/cloud-provider.yml + when: inventory_hostname == groups['kube-master'][0] + tags: oci + +- name: "OCI Cloud Controller | Apply Configuration" + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "/tmp/cloud-provider.yml" + when: inventory_hostname == groups['kube-master'][0] + tags: oci + +- name: "OCI Cloud Controller | Download Controller Manifest" + get_url: + url: "https://raw.githubusercontent.com/oracle/oci-cloud-controller-manager/{{oci_cloud_controller_version}}/manifests/oci-cloud-controller-manager.yaml" + dest: "/tmp/oci-cloud-controller-manager.yml" + force: yes + register: result + until: "'OK' in result.msg" + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: inventory_hostname == groups['kube-master'][0] + tags: oci + +- name: "OCI Cloud Controller | Apply Controller Manifest" + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "/tmp/oci-cloud-controller-manager.yml" + when: inventory_hostname == groups['kube-master'][0] + tags: oci diff --git a/roles/kubernetes-apps/cloud_controller/oci/templates/cloud-provider.yml.j2 b/roles/kubernetes-apps/cloud_controller/oci/templates/cloud-provider.yml.j2 new file mode 100644 index 00000000000..bff3ab43b60 --- /dev/null +++ b/roles/kubernetes-apps/cloud_controller/oci/templates/cloud-provider.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: v1 +data: + cloud-provider.yaml: {{ controller_manager_config_base64 }} +kind: Secret +metadata: + name: oci-cloud-controller-manager + namespace: kube-system +type: Opaque diff --git a/roles/kubernetes-apps/cloud_controller/oci/templates/controller-manager-config.yml.j2 b/roles/kubernetes-apps/cloud_controller/oci/templates/controller-manager-config.yml.j2 new file mode 100644 index 00000000000..38c7ba86ce2 --- /dev/null +++ b/roles/kubernetes-apps/cloud_controller/oci/templates/controller-manager-config.yml.j2 @@ -0,0 +1,56 @@ +auth: + +{% if oci_use_instance_principals %} + # (https://docs.us-phoenix-1.oraclecloud.com/Content/Identity/Tasks/callingservicesfrominstances.htm). + # Ensure you have setup the following OCI policies and your kubernetes nodes are running within them + # allow dynamic-group [your dynamic group name] to read instance-family in compartment [your compartment name] + # allow dynamic-group [your dynamic group name] to use virtual-network-family in compartment [your compartment name] + # allow dynamic-group [your dynamic group name] to manage load-balancers in compartment [your compartment name] + useInstancePrincipals: true +{% else %} + useInstancePrincipals: false + + region: {{ oci_region_id }} + tenancy: {{ oci_tenancy_id }} + user: {{ oci_user_id }} + key: | + {{ oci_private_key }} + + {% if oci_private_key_passphrase is defined %} + passphrase: {{ oci_private_key_passphrase }} + {% endif %} + + + fingerprint: {{ oci_user_fingerprint }} +{% endif %} + +# compartment configures Compartment within which the cluster resides. +compartment: {{ oci_compartment_id }} + +# vcn configures the Virtual Cloud Network (VCN) within which the cluster resides. +vcn: {{ oci_vnc_id }} + +loadBalancer: + # subnet1 configures one of two subnets to which load balancers will be added. + # OCI load balancers require two subnets to ensure high availability. + subnet1: {{ oci_subnet1_id }} + + # subnet2 configures the second of two subnets to which load balancers will be + # added. OCI load balancers require two subnets to ensure high availability. + subnet2: {{ oci_subnet2_id }} + + # SecurityListManagementMode configures how security lists are managed by the CCM. + # "All" (default): Manage all required security list rules for load balancer services. + # "Frontend": Manage only security list rules for ingress to the load + # balancer. Requires that the user has setup a rule that + # allows inbound traffic to the appropriate ports for kube + # proxy health port, node port ranges, and health check port ranges. + # E.g. 10.82.0.0/16 30000-32000. + # "None": Disables all security list management. Requires that the + # user has setup a rule that allows inbound traffic to the + # appropriate ports for kube proxy health port, node port + # ranges, and health check port ranges. E.g. 10.82.0.0/16 30000-32000. + # Additionally requires the user to mange rules to allow + # inbound traffic to load balancers. + securityListManagementMode: {{ oci_security_list_management }} + diff --git a/roles/kubernetes-apps/cluster_roles/defaults/main.yml b/roles/kubernetes-apps/cluster_roles/defaults/main.yml new file mode 100644 index 00000000000..2a99fc0fd7e --- /dev/null +++ b/roles/kubernetes-apps/cluster_roles/defaults/main.yml @@ -0,0 +1,2 @@ +--- +oci_cloud_controller_version: 0.5.0 diff --git a/roles/kubernetes-apps/cluster_roles/tasks/main.yml b/roles/kubernetes-apps/cluster_roles/tasks/main.yml index 3f696a9fec1..229e497e42a 100644 --- a/roles/kubernetes-apps/cluster_roles/tasks/main.yml +++ b/roles/kubernetes-apps/cluster_roles/tasks/main.yml @@ -11,12 +11,54 @@ delay: 6 when: inventory_hostname == groups['kube-master'][0] +- name: Kubernetes Apps | Check AppArmor status + command: which apparmor_parser + register: apparmor_status + when: + - podsecuritypolicy_enabled + - inventory_hostname == groups['kube-master'][0] + failed_when: false + +- name: Kubernetes Apps | Set apparmor_enabled + set_fact: + apparmor_enabled: "{{ apparmor_status.rc == 0 }}" + when: + - podsecuritypolicy_enabled + - inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Apps | Render templates for PodSecurityPolicy + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + register: psp_manifests + with_items: + - {file: psp.yml, type: psp, name: psp} + - {file: psp-cr.yml, type: clusterrole, name: psp-cr} + - {file: psp-crb.yml, type: rolebinding, name: psp-crb} + when: + - podsecuritypolicy_enabled + - inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Apps | Add policies, roles, bindings for PodSecurityPolicy + kube: + name: "{{item.item.name}}" + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: "{{kube_config_dir}}/{{item.item.file}}" + state: "latest" + with_items: "{{ psp_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] + - not item|skipped + - name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes template: src: "node-crb.yml.j2" dest: "{{ kube_config_dir }}/node-crb.yml" register: node_crb_manifest - when: rbac_enabled + when: + - rbac_enabled + - inventory_hostname == groups['kube-master'][0] - name: Apply workaround to allow all nodes with cert O=system:nodes to register kube: @@ -28,6 +70,7 @@ when: - rbac_enabled - node_crb_manifest.changed + - inventory_hostname == groups['kube-master'][0] - name: Kubernetes Apps | Add webhook ClusterRole that grants access to proxy, stats, log, spec, and metrics on a kubelet template: @@ -37,6 +80,7 @@ when: - rbac_enabled - kubelet_authorization_mode_webhook + - inventory_hostname == groups['kube-master'][0] tags: node-webhook - name: Apply webhook ClusterRole @@ -50,6 +94,7 @@ - rbac_enabled - kubelet_authorization_mode_webhook - node_webhook_cr_manifest.changed + - inventory_hostname == groups['kube-master'][0] tags: node-webhook - name: Kubernetes Apps | Add ClusterRoleBinding for system:nodes to webhook ClusterRole @@ -60,6 +105,7 @@ when: - rbac_enabled - kubelet_authorization_mode_webhook + - inventory_hostname == groups['kube-master'][0] tags: node-webhook - name: Grant system:nodes the webhook ClusterRole @@ -73,33 +119,58 @@ - rbac_enabled - kubelet_authorization_mode_webhook - node_webhook_crb_manifest.changed + - inventory_hostname == groups['kube-master'][0] tags: node-webhook -# This is not a cluster role, but should be run after kubeconfig is set on master -- name: Write kube system namespace manifest +- name: Check if vsphere-cloud-provider ClusterRole exists + command: "{{ bin_dir }}/kubectl get clusterroles system:vsphere-cloud-provider" + register: vsphere_cloud_provider + ignore_errors: true + when: + - rbac_enabled + - cloud_provider is defined + - cloud_provider == 'vsphere' + - kube_version | version_compare('v1.9.0', '>=') + - kube_version | version_compare('v1.9.3', '<=') + - inventory_hostname == groups['kube-master'][0] + tags: vsphere + +- name: Write vsphere-cloud-provider ClusterRole manifest template: - src: namespace.j2 - dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml" - when: inventory_hostname == groups['kube-master'][0] - tags: - - apps + src: "vsphere-rbac.yml.j2" + dest: "{{ kube_config_dir }}/vsphere-rbac.yml" + register: vsphere_rbac_manifest + when: + - rbac_enabled + - cloud_provider is defined + - cloud_provider == 'vsphere' + - vsphere_cloud_provider.rc is defined + - vsphere_cloud_provider.rc != 0 + - kube_version | version_compare('v1.9.0', '>=') + - kube_version | version_compare('v1.9.3', '<=') + - inventory_hostname == groups['kube-master'][0] + tags: vsphere -- name: Check if kube system namespace exists - command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}" - register: 'kubesystem' - changed_when: False - failed_when: False - when: inventory_hostname == groups['kube-master'][0] - tags: - - apps +- name: Apply vsphere-cloud-provider ClusterRole + kube: + name: "system:vsphere-cloud-provider" + kubectl: "{{bin_dir}}/kubectl" + resource: "clusterrolebinding" + filename: "{{ kube_config_dir }}/vsphere-rbac.yml" + state: latest + when: + - rbac_enabled + - cloud_provider is defined + - cloud_provider == 'vsphere' + - vsphere_cloud_provider.rc is defined + - vsphere_cloud_provider.rc != 0 + - kube_version | version_compare('v1.9.0', '>=') + - kube_version | version_compare('v1.9.3', '<=') + - inventory_hostname == groups['kube-master'][0] + tags: vsphere -- name: Create kube system namespace - command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml" - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - register: create_system_ns - until: create_system_ns.rc == 0 - changed_when: False - when: inventory_hostname == groups['kube-master'][0] and kubesystem.rc != 0 - tags: - - apps +- include_tasks: oci.yml + tags: oci + when: + - cloud_provider is defined + - cloud_provider == 'oci' diff --git a/roles/kubernetes-apps/cluster_roles/tasks/oci.yml b/roles/kubernetes-apps/cluster_roles/tasks/oci.yml new file mode 100644 index 00000000000..fb89a85e82f --- /dev/null +++ b/roles/kubernetes-apps/cluster_roles/tasks/oci.yml @@ -0,0 +1,23 @@ +--- +- name: Get OCI ClusterRole, and ClusterRoleBinding + get_url: + url: "https://raw.githubusercontent.com/oracle/oci-cloud-controller-manager/{{oci_cloud_controller_version}}/manifests/oci-cloud-controller-manager-rbac.yaml" + dest: "/tmp/oci-cloud-controller-manager-rbac.yaml" + force: yes + register: result + until: "'OK' in result.msg" + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: + - cloud_provider is defined + - cloud_provider == 'oci' + - inventory_hostname == groups['kube-master'][0] + +- name: Apply OCI ClusterRole, and ClusterRoleBinding + kube: + kubectl: "{{bin_dir}}/kubectl" + filename: "/tmp/oci-cloud-controller-manager-rbac.yaml" + when: + - cloud_provider is defined + - cloud_provider == 'oci' + - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 b/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 index 9bdf201a21a..f2e115a6acf 100644 --- a/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 +++ b/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: "{{system_namespace}}" + name: "kube-system" diff --git a/roles/kubernetes-apps/cluster_roles/templates/psp-cr.yml.j2 b/roles/kubernetes-apps/cluster_roles/templates/psp-cr.yml.j2 new file mode 100644 index 00000000000..9fa333ddc62 --- /dev/null +++ b/roles/kubernetes-apps/cluster_roles/templates/psp-cr.yml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:privileged + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - policy + resourceNames: + - privileged + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - policy + resourceNames: + - restricted + resources: + - podsecuritypolicies + verbs: + - use diff --git a/roles/kubernetes-apps/cluster_roles/templates/psp-crb.yml.j2 b/roles/kubernetes-apps/cluster_roles/templates/psp-crb.yml.j2 new file mode 100644 index 00000000000..6cade2883b7 --- /dev/null +++ b/roles/kubernetes-apps/cluster_roles/templates/psp-crb.yml.j2 @@ -0,0 +1,55 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp:any:restricted +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted +subjects: +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: psp:kube-system:privileged + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:privileged +subjects: +- kind: Group + name: system:masters + apiGroup: rbac.authorization.k8s.io +- kind: Group + name: system:serviceaccounts:kube-system + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: psp:nodes:privileged + namespace: kube-system + annotations: + kubernetes.io/description: 'Allow nodes to create privileged pods. Should + be used in combination with the NodeRestriction admission plugin to limit + nodes to mirror pods bound to themselves.' + labels: + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/cluster-service: 'true' +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:privileged +subjects: + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:nodes + - kind: User + apiGroup: rbac.authorization.k8s.io + # Legacy node ID + name: kubelet diff --git a/roles/kubernetes-apps/cluster_roles/templates/psp.yml.j2 b/roles/kubernetes-apps/cluster_roles/templates/psp.yml.j2 new file mode 100644 index 00000000000..a9d32a6e6bb --- /dev/null +++ b/roles/kubernetes-apps/cluster_roles/templates/psp.yml.j2 @@ -0,0 +1,77 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted + annotations: + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' +{% if apparmor_enabled %} + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' +{% endif %} + labels: + kubernetes.io/cluster-service: 'true' + addonmanager.kubernetes.io/mode: Reconcile +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + privileged: true + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + volumes: + - '*' + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + hostIPC: true + hostPID: true + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: false diff --git a/roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2 b/roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2 new file mode 100644 index 00000000000..99da0462f80 --- /dev/null +++ b/roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2 @@ -0,0 +1,35 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:vsphere-cloud-provider +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:vsphere-cloud-provider +roleRef: + kind: ClusterRole + name: system:vsphere-cloud-provider + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: vsphere-cloud-provider + namespace: kube-system diff --git a/roles/kubernetes-apps/container_engine_accelerator/meta/main.yml b/roles/kubernetes-apps/container_engine_accelerator/meta/main.yml new file mode 100644 index 00000000000..c82c5d86bd4 --- /dev/null +++ b/roles/kubernetes-apps/container_engine_accelerator/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: kubernetes-apps/container_engine_accelerator/nvidia_gpu + when: nvidia_accelerator_enabled + tags: + - apps + - nvidia_gpu + - container_engine_accelerator diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml new file mode 100644 index 00000000000..34aea1c473c --- /dev/null +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml @@ -0,0 +1,10 @@ +--- +nvidia_accelerator_enabled: false +nvidia_driver_version: "390.87" +nvidia_gpu_tesla_base_url: https://us.download.nvidia.com/tesla/ +nvidia_gpu_gtx_base_url: http://us.download.nvidia.com/XFree86/Linux-x86_64/ +nvidia_gpu_flavor: tesla +nvidia_url_end: "{{nvidia_driver_version}}/NVIDIA-Linux-x86_64-{{nvidia_driver_version}}.run" +nvidia_driver_install_container: false +nvidia_driver_install_supported: false +nvidia_gpu_nodes: [] diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml new file mode 100644 index 00000000000..50822be7d46 --- /dev/null +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml @@ -0,0 +1,54 @@ +--- + +- name: Container Engine Acceleration Nvidia GPU| gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml" + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml" + - "{{ ansible_distribution|lower }}.yml" + - "{{ ansible_os_family|lower }}.yml" + skip: true + +- name: Container Engine Acceleration Nvidia GPU | Set fact of download url Tesla + set_fact: + nvidia_driver_download_url_default: "{{nvidia_gpu_tesla_base_url}}{{nvidia_url_end}}" + when: nvidia_gpu_flavor|lower == "tesla" + +- name: Container Engine Acceleration Nvidia GPU | Set fact of download url GTX + set_fact: + nvidia_driver_download_url_default: "{{nvidia_gpu_gtx_base_url}}{{nvidia_url_end}}" + when: nvidia_gpu_flavor|lower == "gtx" + +- name: Container Engine Acceleration Nvidia GPU | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/container_engine_accelerator" + owner: root + group: root + mode: 0755 + recurse: true + +- name: Container Engine Acceleration Nvidia GPU | Create manifests for nvidia accelerators + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.file }}" + with_items: + - { name: nvidia-driver-install-daemonset, file: nvidia-driver-install-daemonset.yml, type: daemonset } + - { name: k8s-device-plugin-nvidia-daemonset, file: k8s-device-plugin-nvidia-daemonset.yml, type: daemonset } + register: container_engine_accelerator_manifests + when: + - inventory_hostname == groups['kube-master'][0] and nvidia_driver_install_container + +- name: Container Engine Acceleration Nvidia GPU | Apply manifests for nvidia accelerators + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.item.file }}" + state: "latest" + with_items: + - "{{container_engine_accelerator_manifests.results}}" + when: + - inventory_hostname == groups['kube-master'][0] and nvidia_driver_install_container and nvidia_driver_install_supported diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 new file mode 100644 index 00000000000..84f440442ee --- /dev/null +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-gpu-device-plugin + namespace: kube-system + labels: + k8s-app: nvidia-gpu-device-plugin + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: nvidia-gpu-device-plugin + template: + metadata: + labels: + k8s-app: nvidia-gpu-device-plugin + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-node-critical + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "nvidia.com/gpu" + operator: Exists + tolerations: + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "NoSchedule" + hostNetwork: true + hostPID: true + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins + - name: dev + hostPath: + path: /dev + containers: + - image: "k8s.gcr.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" + command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] + name: nvidia-gpu-device-plugin + resources: + requests: + cpu: 50m + memory: 10Mi + limits: + cpu: 50m + memory: 10Mi + securityContext: + privileged: true + volumeMounts: + - name: device-plugin + mountPath: /device-plugin + - name: dev + mountPath: /dev + updateStrategy: + type: RollingUpdate diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 new file mode 100644 index 00000000000..a1adede5a29 --- /dev/null +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 @@ -0,0 +1,80 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: nvidia-driver-installer + namespace: kube-system +spec: + template: + metadata: + labels: + name: nvidia-driver-installer + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-node-critical + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "nvidia.com/gpu" + operator: Exists + tolerations: + - key: "nvidia.com/gpu" + effect: "NoSchedule" + operator: "Exists" + hostNetwork: true + hostPID: true + volumes: + - name: dev + hostPath: + path: /dev + - name: nvidia-install-dir-host + hostPath: + path: /home/kubernetes/bin/nvidia + - name: root-mount + hostPath: + path: / + initContainers: + - image: "{{nvidia_driver_install_container}}" + name: nvidia-driver-installer + resources: + requests: + cpu: 0.15 + securityContext: + privileged: true + env: + - name: NVIDIA_INSTALL_DIR_HOST + value: /home/kubernetes/bin/nvidia + - name: NVIDIA_INSTALL_DIR_CONTAINER + value: /usr/local/nvidia + - name: ROOT_MOUNT_DIR + value: /root + - name: NVIDIA_DRIVER_VERSION + value: "{{nvidia_driver_version}}" + - name: NVIDIA_DRIVER_DOWNLOAD_URL + value: "{{nvidia_driver_download_url_default}}" + volumeMounts: + - name: nvidia-install-dir-host + mountPath: /usr/local/nvidia + - name: dev + mountPath: /dev + - name: root-mount + mountPath: /root + containers: + - image: "gcr.io/google-containers/pause:2.0" + name: pause diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/centos-7.yml b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/centos-7.yml new file mode 100644 index 00000000000..5f6adfde759 --- /dev/null +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/centos-7.yml @@ -0,0 +1,3 @@ +--- +nvidia_driver_install_container: atzedevries/nvidia-centos-driver-installer:2 +nvidia_driver_install_supported: true diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml new file mode 100644 index 00000000000..04b9e0ac9be --- /dev/null +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml @@ -0,0 +1,3 @@ +--- +nvidia_driver_install_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:eea7309dc4fa4a5c9d716157e74b90826e0a853aa26c7219db4710ddcd1ad8bc +nvidia_driver_install_supported: true diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml new file mode 100644 index 00000000000..04b9e0ac9be --- /dev/null +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml @@ -0,0 +1,3 @@ +--- +nvidia_driver_install_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:eea7309dc4fa4a5c9d716157e74b90826e0a853aa26c7219db4710ddcd1ad8bc +nvidia_driver_install_supported: true diff --git a/roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml b/roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml index 8abbe231711..888cbd189ce 100644 --- a/roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml +++ b/roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml @@ -7,15 +7,13 @@ - "efk-sa.yml" - "efk-clusterrolebinding.yml" run_once: true - when: rbac_enabled - name: "ElasticSearch | Create Serviceaccount and Clusterrolebinding (RBAC)" - command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/{{ item }} -n {{ system_namespace }}" + command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/{{ item }} -n kube-system" with_items: - "efk-sa.yml" - "efk-clusterrolebinding.yml" run_once: true - when: rbac_enabled - name: "ElasticSearch | Write ES deployment" template: @@ -24,7 +22,7 @@ register: es_deployment_manifest - name: "ElasticSearch | Create ES deployment" - command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/elasticsearch-deployment.yaml -n {{ system_namespace }}" + command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/elasticsearch-deployment.yaml -n kube-system" run_once: true when: es_deployment_manifest.changed @@ -35,6 +33,6 @@ register: es_service_manifest - name: "ElasticSearch | Create ES service" - command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/elasticsearch-service.yaml -n {{ system_namespace }}" + command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/elasticsearch-service.yaml -n kube-system" run_once: true when: es_service_manifest.changed diff --git a/roles/kubernetes-apps/efk/elasticsearch/templates/efk-clusterrolebinding.yml b/roles/kubernetes-apps/efk/elasticsearch/templates/efk-clusterrolebinding.yml index a5aba61aef5..4b9ab006737 100644 --- a/roles/kubernetes-apps/efk/elasticsearch/templates/efk-clusterrolebinding.yml +++ b/roles/kubernetes-apps/efk/elasticsearch/templates/efk-clusterrolebinding.yml @@ -1,13 +1,16 @@ --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: efk - namespace: {{ system_namespace }} + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile subjects: - kind: ServiceAccount name: efk - namespace: {{ system_namespace }} + namespace: kube-system roleRef: kind: ClusterRole name: cluster-admin diff --git a/roles/kubernetes-apps/efk/elasticsearch/templates/efk-sa.yml b/roles/kubernetes-apps/efk/elasticsearch/templates/efk-sa.yml index e79e26be87f..01e774e9666 100644 --- a/roles/kubernetes-apps/efk/elasticsearch/templates/efk-sa.yml +++ b/roles/kubernetes-apps/efk/elasticsearch/templates/efk-sa.yml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: efk - namespace: {{ system_namespace }} + namespace: kube-system labels: kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 b/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 index 6d5382e09ad..ad1adc536c6 100644 --- a/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 +++ b/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 @@ -1,15 +1,17 @@ --- -# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-elasticsearch/es-controller.yaml -apiVersion: extensions/v1beta1 -kind: Deployment +# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.10.2/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet metadata: - name: elasticsearch-logging-v1 - namespace: "{{ system_namespace }}" + name: elasticsearch-logging + namespace: kube-system labels: k8s-app: elasticsearch-logging version: "{{ elasticsearch_image_tag }}" kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile spec: + serviceName: elasticsearch-logging replicas: 2 selector: matchLabels: @@ -30,12 +32,12 @@ spec: limits: cpu: {{ elasticsearch_cpu_limit }} {% if elasticsearch_mem_limit is defined and elasticsearch_mem_limit != "0M" %} - mem: {{ elasticsearch_mem_limit }} + memory: "{{ elasticsearch_mem_limit }}" {% endif %} requests: cpu: {{ elasticsearch_cpu_requests }} {% if elasticsearch_mem_requests is defined and elasticsearch_mem_requests != "0M" %} - mem: {{ elasticsearch_mem_requests }} + memory: "{{ elasticsearch_mem_requests }}" {% endif %} ports: - containerPort: 9200 @@ -50,7 +52,11 @@ spec: volumes: - name: es-persistent-storage emptyDir: {} -{% if rbac_enabled %} serviceAccountName: efk -{% endif %} + initContainers: + - image: alpine:3.6 + command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"] + name: elasticsearch-logging-init + securityContext: + privileged: true diff --git a/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 b/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 index b7558f9d938..789ecb215a0 100644 --- a/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 +++ b/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Service metadata: name: elasticsearch-logging - namespace: "{{ system_namespace }}" + namespace: "kube-system" labels: k8s-app: elasticsearch-logging kubernetes.io/cluster-service: "true" diff --git a/roles/kubernetes-apps/efk/fluentd/defaults/main.yml b/roles/kubernetes-apps/efk/fluentd/defaults/main.yml index e8d93732c57..0305a5f7a7c 100644 --- a/roles/kubernetes-apps/efk/fluentd/defaults/main.yml +++ b/roles/kubernetes-apps/efk/fluentd/defaults/main.yml @@ -1,7 +1,7 @@ --- fluentd_cpu_limit: 0m -fluentd_mem_limit: 200Mi +fluentd_mem_limit: 500Mi fluentd_cpu_requests: 100m fluentd_mem_requests: 200Mi -fluentd_config_dir: /etc/kubernetes/fluentd -fluentd_config_file: fluentd.conf +fluentd_config_dir: /etc/fluent/config.d +# fluentd_config_file: fluentd.conf diff --git a/roles/kubernetes-apps/efk/fluentd/tasks/main.yml b/roles/kubernetes-apps/efk/fluentd/tasks/main.yml index c91bf68276e..f444c79b62f 100644 --- a/roles/kubernetes-apps/efk/fluentd/tasks/main.yml +++ b/roles/kubernetes-apps/efk/fluentd/tasks/main.yml @@ -17,6 +17,6 @@ register: fluentd_ds_manifest - name: "Fluentd | Create fluentd daemonset" - command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/fluentd-ds.yaml -n {{ system_namespace }}" + command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/fluentd-ds.yaml -n kube-system" run_once: true when: fluentd_ds_manifest.changed diff --git a/roles/kubernetes-apps/efk/fluentd/templates/fluentd-config.yml.j2 b/roles/kubernetes-apps/efk/fluentd/templates/fluentd-config.yml.j2 index 8a8ebbceca8..0b0229f69fd 100644 --- a/roles/kubernetes-apps/efk/fluentd/templates/fluentd-config.yml.j2 +++ b/roles/kubernetes-apps/efk/fluentd/templates/fluentd-config.yml.j2 @@ -1,10 +1,19 @@ +--- +# https://raw.githubusercontent.com/kubernetes/kubernetes/release-1.10/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: fluentd-config - namespace: "{{ system_namespace }}" + namespace: "kube-system" + labels: + addonmanager.kubernetes.io/mode: Reconcile data: - {{ fluentd_config_file }}: | + system.conf: |- + + root_dir /tmp/fluentd-buffers/ + + + containers.input.conf: |- # This configuration file for Fluentd / td-agent is used # to watch changes to Docker log files. The kubelet creates symlinks that # capture the pod name, namespace, container name & Docker container ID @@ -18,7 +27,6 @@ data: # See https://github.com/uken/fluent-plugin-elasticsearch & # https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter for # more information about the plugins. - # Maintainer: Jimmi Dyson # # Example # ======= @@ -99,63 +107,87 @@ data: # This makes it easier for users to search for logs by pod name or by # the name of the Kubernetes container regardless of how many times the # Kubernetes pod has been restarted (resulting in a several Docker container IDs). - # - # TODO: Propagate the labels associated with a container along with its logs - # so users can query logs using labels as well as or instead of the pod name - # and container name. This is simply done via configuration of the Kubernetes - # fluentd plugin but requires secrets to be enabled in the fluent pod. This is a - # problem yet to be solved as secrets are not usable in static pods which the fluentd - # pod must be until a per-node controller is available in Kubernetes. - # Prevent fluentd from handling records containing its own logs. Otherwise - # it can lead to an infinite loop, when error in sending one message generates - # another message which also fails to be sent and so on. - - type null - - # Example: + + # Json Log Example: # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} + # CRI Log Example: + # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here - type tail + @id fluentd-containers.log + @type tail path /var/log/containers/*.log pos_file /var/log/es-containers.log.pos time_format %Y-%m-%dT%H:%M:%S.%NZ - tag kubernetes.* - format json + tag raw.kubernetes.* read_from_head true + + @type multi_format + + format json + time_key time + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format /^(? + + + # Detect exceptions in the log output and forward them as one log entry. + + @id raw.kubernetes + @type detect_exceptions + remove_tag_prefix raw + message log + stream stream + multiline_flush_interval 5 + max_bytes 500000 + max_lines 1000 + + + system.input.conf: |- # Example: # 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 - type tail + @id minion + @type tail format /^(?