From 280096fce4053579b4baf1c4808a6aecd95af054 Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Wed, 4 Jan 2017 17:32:16 +0100 Subject: [PATCH 1/4] [cern] Upgrade to Fedora Atomic 25 Cherry-pick: https://review.openstack.org/#/c/417457/8 Image contains: kubernetes-1.4.7-1.fc25.x86_64 docker-1.12.5-4.git03508cc.fc25.x86_64 flannel-0.5.5-8.fc25.x86_64 etcd-3.0.15-1.fc25.x86_64 * For this upgrade the upstream image is used, which is uploaded here [1]. * Minor changes for flannel and docker-storage-setup were needed. * The image will be built in the CI and uploaded to tarballs.openstack.org as soon as possible. [1] https://fedorapeople.org/groups/magnum/fedora-atomic-25-20161221.qcow2 Change-Id: Iac6e30c530821a49a5c3978e335e0b1d56a576e0 Conflicts: magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh magnum/drivers/common/templates/kubernetes/fragments/write-network-config.sh magnum/drivers/k8s_fedora_atomic_v1/version.py magnum/drivers/swarm_fedora_atomic_v1/version.py magnum/tests/contrib/gate_hook.sh --- .../fragments/configure-docker-storage.sh | 6 ++++++ .../configure_docker_storage_driver_atomic.sh | 15 ++++++++------- .../fragments/configure-kubernetes-minion.sh | 4 ++-- .../fragments/network-config-service.sh | 4 ++-- .../kubernetes/fragments/write-network-config.sh | 2 +- .../templates/kubecluster.yaml | 2 +- magnum/drivers/k8s_fedora_atomic_v1/version.py | 4 ++-- .../templates/kubecluster.yaml | 2 +- .../templates/fragments/enable-services.sh | 5 +++++ magnum/drivers/swarm_fedora_atomic_v1/version.py | 4 ++-- magnum/tests/contrib/copy_instance_logs.sh | 3 +++ magnum/tests/contrib/gate_hook.sh | 8 ++++---- 12 files changed, 37 insertions(+), 22 deletions(-) diff --git a/magnum/drivers/common/templates/fragments/configure-docker-storage.sh b/magnum/drivers/common/templates/fragments/configure-docker-storage.sh index 104c3ac5e..2b22ab23e 100644 --- a/magnum/drivers/common/templates/fragments/configure-docker-storage.sh +++ b/magnum/drivers/common/templates/fragments/configure-docker-storage.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -x + . /etc/sysconfig/heat-params if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then @@ -30,6 +32,8 @@ if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then fi fi +systemctl stop docker + $configure_docker_storage_driver if [ "$DOCKER_STORAGE_DRIVER" = "overlay" ]; then @@ -44,3 +48,5 @@ if [ "$DOCKER_STORAGE_DRIVER" = "overlay" ]; then else configure_devicemapper fi + +systemctl start docker-storage-setup diff --git a/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh b/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh index a28dc8f73..4785cabd8 100644 --- a/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh +++ b/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh @@ -1,19 +1,20 @@ # This file contains docker storage drivers configuration for fedora # atomic hosts. Currently, devicemapper and overlay are supported. -# Remove any existing docker-storage configuration. In case of an -# existing configuration, docker-storage-setup will fail. -clear_docker_storage_congiguration () { +# - Remove any existing docker-storage configuration. In case of an +# existing configuration, docker-storage-setup will fail. +# - Remove the current storage graph. +clear_docker_storage () { if [ -f /etc/sysconfig/docker-storage ]; then sed -i "/^DOCKER_STORAGE_OPTIONS=/ s/=.*/=/" /etc/sysconfig/docker-storage fi + + rm -rf /var/lib/docker/* } # Configure docker storage with xfs as backing filesystem. configure_overlay () { - clear_docker_storage_congiguration - - rm -rf /var/lib/docker/* + clear_docker_storage if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then mkfs.xfs -f ${device_path} @@ -31,7 +32,7 @@ configure_overlay () { # Configure docker storage with devicemapper using direct LVM configure_devicemapper () { - clear_docker_storage_congiguration + clear_docker_storage if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then pvcreate -f ${device_path} diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 2ce5a8afb..745709189 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -73,13 +73,13 @@ sed -i ' if [ "$NETWORK_DRIVER" = "flannel" ]; then sed -i ' - /^FLANNEL_ETCD=/ s|=.*|="'"$PROTOCOL"'://'"$ETCD_SERVER_IP"':2379"| + /^FLANNEL_ETCD_ENDPOINTS=/ s|=.*|="'"$PROTOCOL"'://'"$ETCD_SERVER_IP"':2379"| ' $FLANNELD_CONFIG # Make sure etcd has a flannel configuration . $FLANNELD_CONFIG until curl -sf $ETCD_CURL_OPTIONS \ - "$FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config?quorum=false&recursive=false&sorted=false" + "$FLANNEL_ETCD_ENDPOINTS/v2/keys${FLANNEL_ETCD_PREFIX}/config?quorum=false&recursive=false&sorted=false" do echo "Waiting for flannel configuration in etcd..." sleep 5 diff --git a/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh index 1cac6b580..643179cf2 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh @@ -41,14 +41,14 @@ if ! [ -f "$FLANNEL_JSON" ]; then exit 1 fi -if ! [ "$FLANNEL_ETCD" ] && [ "$FLANNEL_ETCD_KEY" ]; then +if [ -z "$FLANNEL_ETCD_ENDPOINTS" ] || [ -z "$FLANNEL_ETCD_PREFIX" ]; then echo "ERROR: missing required configuration" >&2 exit 1 fi echo "creating flanneld config in etcd" while ! curl -sf -L $ETCD_CURL_OPTIONS \ - $FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \ + $FLANNEL_ETCD_ENDPOINTS/v2/keys${FLANNEL_ETCD_PREFIX}/config \ -X PUT --data-urlencode value@${FLANNEL_JSON}; do echo "waiting for etcd" sleep 1 diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-network-config.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-network-config.sh index 1fcadcded..3b2c54b1b 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-network-config.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-network-config.sh @@ -12,7 +12,7 @@ FLANNEL_JSON=/etc/sysconfig/flannel-network.json FLANNELD_CONFIG=/etc/sysconfig/flanneld sed -i ' - /^FLANNEL_ETCD=/ s/=.*/="http:\/\/127.0.0.1:2379"/ + /^FLANNEL_ETCD_ENDPOINTS=/ s/=.*/="http:\/\/127.0.0.1:2379"/ ' /etc/sysconfig/flanneld # Generate a flannel configuration that we will diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 3befb49c6..692d9b093 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -256,7 +256,7 @@ parameters: kube_version: type: string description: version of kubernetes used for kubernetes cluster - default: v1.2.0 + default: v1.4.7 insecure_registry_url: type: string diff --git a/magnum/drivers/k8s_fedora_atomic_v1/version.py b/magnum/drivers/k8s_fedora_atomic_v1/version.py index 7f1702c33..ebe1bbafa 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/version.py +++ b/magnum/drivers/k8s_fedora_atomic_v1/version.py @@ -13,5 +13,5 @@ # limitations under the License. version = '1.0.0' -driver = 'k8s_fedora_atomic' -container_version = '1.9.1' +driver = 'k8s_fedora_atomic_v1' +container_version = '1.12.5' diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml index baec083fc..e95cf100d 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml @@ -269,7 +269,7 @@ parameters: kube_version: type: string description: version of kubernetes used for kubernetes cluster - default: v1.2.0 + default: v1.4.7 insecure_registry_url: type: string diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/enable-services.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/enable-services.sh index 1c7ed9790..79d12626e 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/enable-services.sh +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/enable-services.sh @@ -1,5 +1,10 @@ #!/bin/sh +set -x + +echo "stopping docker" +systemctl stop docker + echo "starting services" systemctl daemon-reload for service in $NODE_SERVICES; do diff --git a/magnum/drivers/swarm_fedora_atomic_v1/version.py b/magnum/drivers/swarm_fedora_atomic_v1/version.py index a36bff97b..09cd1519d 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/version.py +++ b/magnum/drivers/swarm_fedora_atomic_v1/version.py @@ -13,5 +13,5 @@ # limitations under the License. version = '1.0.0' -driver = 'swarm_atomic' -container_version = '1.9.1' +driver = 'swarm_fedora_atomic_v1' +container_version = '1.12.5' diff --git a/magnum/tests/contrib/copy_instance_logs.sh b/magnum/tests/contrib/copy_instance_logs.sh index a12db1a4d..ebb9e3f5c 100755 --- a/magnum/tests/contrib/copy_instance_logs.sh +++ b/magnum/tests/contrib/copy_instance_logs.sh @@ -81,6 +81,7 @@ elif [[ "$COE" == "swarm" ]]; then remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log + remote_exec $SSH_USER "sudo cat /var/log/cloud-init-output.log" cloud-init-output.log remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log remote_exec $SSH_USER "sudo journalctl -u swarm-manager --no-pager" swarm-manager.log remote_exec $SSH_USER "sudo journalctl -u swarm-agent --no-pager" swarm-agent.log @@ -89,6 +90,8 @@ elif [[ "$COE" == "swarm" ]]; then remote_exec $SSH_USER "sudo systemctl show docker-storage-setup --no-pager" docker-storage-setup.service.show.log remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage-setup 2>/dev/null" docker-storage-setup.sysconfig.env.log remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log + remote_exec $SSH_USER "sudo systemctl status docker.socket -l" docker.socket.status.log + remote_exec $SSH_USER "sudo systemctl show docker.socket --no-pager" docker.socket.show.log remote_exec $SSH_USER "sudo systemctl status docker -l" docker.service.status.log remote_exec $SSH_USER "sudo systemctl show docker --no-pager" docker.service.show.log remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log diff --git a/magnum/tests/contrib/gate_hook.sh b/magnum/tests/contrib/gate_hook.sh index d88e9c260..5b8c3098a 100755 --- a/magnum/tests/contrib/gate_hook.sh +++ b/magnum/tests/contrib/gate_hook.sh @@ -32,8 +32,8 @@ if [ "$coe" = "mesos" ]; then elif [ "$coe" = "k8s-coreos" ]; then echo "MAGNUM_GUEST_IMAGE_URL=http://beta.release.core-os.net/amd64-usr/1153.4.0/coreos_production_openstack_image.img.bz2" >> $BASE/new/devstack/localrc elif [ "${coe}${special}" = "k8s-ironic" ]; then - export DEVSTACK_LOCAL_CONFIG+=$'\n'"MAGNUM_GUEST_IMAGE_URL='https://fedorapeople.org/groups/magnum/fedora-24-kubernetes-ironic.tar.gz'" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"MAGNUM_IMAGE_NAME='fedora-24-kubernetes-ironic'" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"MAGNUM_GUEST_IMAGE_URL='https://fedorapeople.org/groups/magnum/fedora-25-kubernetes-ironic.tar.gz'" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"MAGNUM_IMAGE_NAME='fedora-25-kubernetes-ironic'" export DEVSTACK_GATE_VIRT_DRIVER="ironic" # NOTE(yuanying): Current implementation requires only 1 subnet under network @@ -78,8 +78,8 @@ elif [ "${coe}${special}" = "k8s-ironic" ]; then export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_SPECS_DISK=10" export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_EPHEMERAL_DISK=5" else - export DEVSTACK_LOCAL_CONFIG+=$'\n'"MAGNUM_GUEST_IMAGE_URL='http://tarballs.openstack.org/magnum/images/fedora-atomic-f23-dib.qcow2'" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"MAGNUM_IMAGE_NAME='fedora-atomic-f23-dib'" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"MAGNUM_GUEST_IMAGE_URL='https://fedorapeople.org/groups/magnum/fedora-atomic-25-20170106.qcow2'" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"MAGNUM_IMAGE_NAME='fedora-atomic-25-20170106'" fi # Enable magnum plugin in the last step -- GitLab From 29798308e822a833c884d4690e3c6754c8a2cef2 Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Wed, 30 Nov 2016 15:20:28 +0100 Subject: [PATCH 2/4] [cern] Add docker-d options in sysconfig/docker Cherry-pick: https://review.openstack.org/#/c/404789/7 Remove custom docker unit file and pass the necessary options through /etc/sysconfig/docker file. Change-Id: I6bf91843b9120b700d13aad54cef38342ae1f8bd Closes-Bug: #1646123 Conflicts: magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-service.sh magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml --- .../fragments/add-docker-daemon-options.sh | 13 +++++ .../fragments/write-docker-service.sh | 52 ------------------- .../fragments/write-docker-socket.yaml | 1 + .../templates/swarmmaster.yaml | 6 +-- .../templates/swarmnode.yaml | 6 +-- 5 files changed, 20 insertions(+), 58 deletions(-) create mode 100644 magnum/drivers/common/templates/swarm/fragments/add-docker-daemon-options.sh delete mode 100644 magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-service.sh diff --git a/magnum/drivers/common/templates/swarm/fragments/add-docker-daemon-options.sh b/magnum/drivers/common/templates/swarm/fragments/add-docker-daemon-options.sh new file mode 100644 index 000000000..b08eb3132 --- /dev/null +++ b/magnum/drivers/common/templates/swarm/fragments/add-docker-daemon-options.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. /etc/sysconfig/heat-params + +opts="-H fd:// -H tcp://0.0.0.0:2375 " + +if [ "$TLS_DISABLED" = 'False' ]; then + opts=$opts"--tlsverify --tlscacert=/etc/docker/ca.crt " + opts=$opts"--tlskey=/etc/docker/server.key " + opts=$opts"--tlscert=/etc/docker/server.crt " +fi + +sed -i '/^OPTIONS=/ s#\(OPTIONS='"'"'\)#\1'"$opts"'#' /etc/sysconfig/docker diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-service.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-service.sh deleted file mode 100644 index f8b35bd10..000000000 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-service.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh - -. /etc/sysconfig/heat-params - -mkdir -p /etc/systemd/system/docker.service.d - -cat > /etc/systemd/system/docker.service << END_SERVICE_TOP -[Unit] -Description=Docker Application Container Engine -Documentation=http://docs.docker.com -After=network.target docker.socket -Requires=docker.socket -Wants=docker-storage-setup.service - -[Service] -TimeoutStartSec=300 -Type=notify -EnvironmentFile=-/etc/sysconfig/docker -EnvironmentFile=-/etc/sysconfig/docker-storage -EnvironmentFile=-/etc/sysconfig/docker-network -Environment=GOTRACEBACK=crash -ExecStart=/usr/bin/docker daemon -H fd:// \\ - -H tcp://0.0.0.0:2375 \\ -END_SERVICE_TOP - -if [ "$TLS_DISABLED" = 'False' ]; then - -cat >> /etc/systemd/system/docker.service << END_TLS - --tlsverify \\ - --tlscacert="/etc/docker/ca.crt" \\ - --tlskey="/etc/docker/server.key" \\ - --tlscert="/etc/docker/server.crt" \\ -END_TLS - -fi - -cat >> /etc/systemd/system/docker.service << END_SERVICE_BOTTOM - \$OPTIONS \\ - \$DOCKER_STORAGE_OPTIONS \\ - \$DOCKER_NETWORK_OPTIONS \\ - \$INSECURE_REGISTRY -LimitNOFILE=1048576 -LimitNPROC=1048576 -LimitCORE=infinity -MountFlags=slave - -[Install] -WantedBy=multi-user.target -END_SERVICE_BOTTOM - -chown root:root /etc/systemd/system/docker.service -chmod 644 /etc/systemd/system/docker.service diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-socket.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-socket.yaml index 59cf0991c..4c1b9fcd0 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-socket.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-socket.yaml @@ -9,6 +9,7 @@ write_files: Description=Docker Socket for the API PartOf=docker.service After=docker-storage-setup.service + Before=docker.service [Socket] ListenStream=/var/run/docker.sock diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index 70fcb247a..72dc5b0bd 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -255,11 +255,11 @@ resources: group: ungrouped config: {get_file: fragments/make-cert.py} - write_docker_service: + add_docker_daemon_options: type: "OS::Heat::SoftwareConfig" properties: group: ungrouped - config: {get_file: fragments/write-docker-service.sh} + config: {get_file: ../../common/templates/swarm/fragments/add-docker-daemon-options.sh} write_swarm_manager_failure_service: type: "OS::Heat::SoftwareConfig" @@ -345,7 +345,7 @@ resources: - config: {get_resource: network_service} - config: {get_resource: configure_docker_storage} - config: {get_resource: write_swarm_manager_failure_service} - - config: {get_resource: write_docker_service} + - config: {get_resource: add_docker_daemon_options} - config: {get_resource: write_docker_socket} - config: {get_resource: write_swarm_master_service} - config: {get_resource: add_proxy} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index ea61a069f..507fecb7b 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -246,11 +246,11 @@ resources: group: ungrouped config: {get_file: ../../common/templates/fragments/configure-docker-registry.sh} - write_docker_service: + add_docker_daemon_options: type: "OS::Heat::SoftwareConfig" properties: group: ungrouped - config: {get_file: fragments/write-docker-service.sh} + config: {get_file: ../../common/templates/swarm/fragments/add-docker-daemon-options.sh} write_docker_socket: type: "OS::Heat::SoftwareConfig" @@ -341,7 +341,7 @@ resources: - config: {get_resource: configure_docker_registry} - config: {get_resource: write_swarm_agent_failure_service} - config: {get_resource: write_swarm_agent_service} - - config: {get_resource: write_docker_service} + - config: {get_resource: add_docker_daemon_options} - config: {get_resource: write_docker_socket} - config: {get_resource: add_proxy} - config: {get_resource: enable_docker_registry} -- GitLab From decea7aa34dee133d596daf135138b7807857095 Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Sat, 26 Nov 2016 17:10:00 +0100 Subject: [PATCH 3/4] [cern] [k8s_fedora_atomic] Remove podmaster Cherry-pick: https://review.openstack.org/#/c/404782/ Podmaster is deprecated since k8s 1.2 and its docker image is v1, incompatible with docker >=1.12. * Remove podmaster pod * Update manifests of kube-controller-manager and kube-scheduler * Rename SoftwareConfig to reflect the new functionality Closes-Bug: #1646109 Change-Id: Ibf4ce06cbf5b79a4241c58c67b13a7c68145d3ae Conflicts: magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml --- ...able-kube-controller-manager-scheduler.sh} | 58 +------------------ .../templates/kubemaster.yaml | 6 +- .../templates/kubemaster.yaml | 6 +- 3 files changed, 8 insertions(+), 62 deletions(-) rename magnum/drivers/common/templates/kubernetes/fragments/{enable-kube-podmaster.sh => enable-kube-controller-manager-scheduler.sh} (58%) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-kube-podmaster.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-kube-controller-manager-scheduler.sh similarity index 58% rename from magnum/drivers/common/templates/kubernetes/fragments/enable-kube-podmaster.sh rename to magnum/drivers/common/templates/kubernetes/fragments/enable-kube-controller-manager-scheduler.sh index 5aa1bfaba..3474ea8e1 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-kube-podmaster.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-kube-controller-manager-scheduler.sh @@ -3,67 +3,13 @@ . /etc/sysconfig/heat-params if [ -n "${INSECURE_REGISTRY_URL}" ]; then - PODMASTER_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/podmaster:1.1" HYPERKUBE_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/hyperkube:${KUBE_VERSION}" else - PODMASTER_IMAGE="gcr.io/google_containers/podmaster:1.1" HYPERKUBE_IMAGE="gcr.io/google_containers/hyperkube:${KUBE_VERSION}" fi init_templates () { - local TEMPLATE=/etc/kubernetes/manifests/kube-podmaster.yaml - [ -f ${TEMPLATE} ] || { - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname ${TEMPLATE}) - cat << EOF > ${TEMPLATE} -apiVersion: v1 -kind: Pod -metadata: - name: kube-podmaster - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: scheduler-elector - image: ${PODMASTER_IMAGE} - command: - - /podmaster - - --etcd-servers=http://127.0.0.1:2379 - - --key=scheduler - - --source-file=/src/manifests/kube-scheduler.yaml - - --dest-file=/dst/manifests/kube-scheduler.yaml - volumeMounts: - - mountPath: /src/manifests - name: manifest-src - readOnly: true - - mountPath: /dst/manifests - name: manifest-dst - - name: controller-manager-elector - image: ${PODMASTER_IMAGE} - command: - - /podmaster - - --etcd-servers=http://127.0.0.1:2379 - - --key=controller - - --source-file=/src/manifests/kube-controller-manager.yaml - - --dest-file=/dst/manifests/kube-controller-manager.yaml - terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /src/manifests - name: manifest-src - readOnly: true - - mountPath: /dst/manifests - name: manifest-dst - volumes: - - hostPath: - path: /srv/kubernetes/manifests - name: manifest-src - - hostPath: - path: /etc/kubernetes/manifests - name: manifest-dst -EOF - } - local SERVICE_ACCOUNT_PRIVATE_KEY_FILE=/etc/kubernetes/ssl/server.key local ROOT_CA_FILE=/etc/kubernetes/ssl/ca.crt @@ -72,7 +18,7 @@ EOF ROOT_CA_FILE= fi - local TEMPLATE=/srv/kubernetes/manifests/kube-controller-manager.yaml + local TEMPLATE=/etc/kubernetes/manifests/kube-controller-manager.yaml [ -f ${TEMPLATE} ] || { echo "TEMPLATE: $TEMPLATE" mkdir -p $(dirname ${TEMPLATE}) @@ -124,7 +70,7 @@ spec: EOF } - local TEMPLATE=/srv/kubernetes/manifests/kube-scheduler.yaml + local TEMPLATE=/etc/kubernetes/manifests/kube-scheduler.yaml [ -f ${TEMPLATE} ] || { echo "TEMPLATE: $TEMPLATE" mkdir -p $(dirname ${TEMPLATE}) diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 8b4cf6eba..c6ea154dc 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -306,11 +306,11 @@ resources: group: ungrouped config: {get_file: ../../common/templates/kubernetes/fragments/network-service.sh} - enable_kube_podmaster: + enable_kube_controller_manager_scheduler: type: OS::Heat::SoftwareConfig properties: group: ungrouped - config: {get_file: ../../common/templates/kubernetes/fragments/enable-kube-podmaster.sh} + config: {get_file: ../../common/templates/kubernetes/fragments/enable-kube-controller-manager-scheduler.sh} kube_system_namespace_service: type: OS::Heat::SoftwareConfig @@ -378,7 +378,7 @@ resources: - config: {get_resource: network_service} - config: {get_resource: kube_system_namespace_service} - config: {get_resource: kube_dns_service} - - config: {get_resource: enable_kube_podmaster} + - config: {get_resource: enable_kube_controller_manager_scheduler} - config: {get_resource: enable_kube_proxy} - config: {get_resource: kube_ui_service} - config: {get_resource: kube_examples} diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml index 27f4fe2e5..1b1f1d1f8 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml @@ -319,11 +319,11 @@ resources: group: ungrouped config: {get_file: ../../common/templates/kubernetes/fragments/network-service.sh} - enable_kube_podmaster: + enable_kube_controller_manager_scheduler: type: OS::Heat::SoftwareConfig properties: group: ungrouped - config: {get_file: ../../common/templates/kubernetes/fragments/enable-kube-podmaster.sh} + config: {get_file: ../../common/templates/kubernetes/fragments/enable-kube-controller-manager-scheduler.sh} kube_system_namespace_service: type: OS::Heat::SoftwareConfig @@ -378,7 +378,7 @@ resources: - config: {get_resource: network_config_service} - config: {get_resource: network_service} - config: {get_resource: kube_system_namespace_service} - - config: {get_resource: enable_kube_podmaster} + - config: {get_resource: enable_kube_controller_manager_scheduler} - config: {get_resource: enable_kube_proxy} - config: {get_resource: kube_ui_service} - config: {get_resource: kube_examples} -- GitLab From 1d1ad781c4d4eb43d0f2be2d57235b47a499f92a Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Tue, 10 Jan 2017 14:17:02 +0100 Subject: [PATCH 4/4] [cern] add cvmfs container configuration add systemd setup to configure cvmfs via docker-volume-cvmfs. add new labels to configure cvmfs: * cvmfs_enabled: if cvmfs configuration should be done (default true) * cvmfs_tag: which docker-volume-cvmfs tag to use (default latest) Implements OS-3825. --- .../templates/fragments/configure-cvmfs.sh | 25 +++++++++++++++++++ .../kubernetes/fragments/configure-cvmfs.sh | 25 +++++++++++++++++++ .../fragments/configure-kubernetes-minion.sh | 2 +- .../fragments/write-heat-params.yaml | 2 ++ .../templates/kubecluster.yaml | 14 +++++++++++ .../templates/kubeminion.yaml | 21 ++++++++++++++++ .../swarm_fedora_atomic_v1/template_def.py | 3 ++- .../templates/cluster.yaml | 14 +++++++++++ .../fragments/write-heat-params-node.sh | 2 ++ .../templates/swarmnode.yaml | 23 ++++++++++++++++- 10 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 magnum/drivers/common/templates/fragments/configure-cvmfs.sh create mode 100644 magnum/drivers/common/templates/kubernetes/fragments/configure-cvmfs.sh diff --git a/magnum/drivers/common/templates/fragments/configure-cvmfs.sh b/magnum/drivers/common/templates/fragments/configure-cvmfs.sh new file mode 100644 index 000000000..fd2c8c669 --- /dev/null +++ b/magnum/drivers/common/templates/fragments/configure-cvmfs.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +. /etc/sysconfig/heat-params + +if [ "$CVMFS_ENABLED" = "False" ]; then + exit 0 +fi + +chattr -i / +mkdir /cvmfs +chattr +i / + +atomic install gitlab-registry.cern.ch/cloud-infrastructure/docker-volume-cvmfs:${CVMFS_TAG} + +# add selinux policy +docker cp docker-volume-cvmfs:/dockercvmfs.pp /tmp +semodule -i /tmp/dockercvmfs.pp + +# install kubernetes volume plugin +mkdir -p /var/lib/kubelet/plugins/volume/exec/cern~cvmfs +docker cp docker-volume-cvmfs:/usr/sbin/docker-volume-cvmfs /var/lib/kubelet/plugins/volume/exec/cern~cvmfs/cvmfs + +# TODO: move this elsewhere +lvextend /dev/atomicos/root --size 5G +xfs_growfs /dev/mapper/atomicos-root diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-cvmfs.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-cvmfs.sh new file mode 100644 index 000000000..20bdb0374 --- /dev/null +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-cvmfs.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +. /etc/sysconfig/heat-params + +if [ "$CVMFS_ENABLED" = "False" ]; then + exit 0 +fi + +chattr -i / +mkdir /cvmfs +chattr +i / + +atomic install gitlab-registry.cern.ch/cloud-infrastructure/docker-volume-cvmfs:${CVMFS_TAG} + +# add selinux policy +docker cp docker-volume-cvmfs:/dockercvmfs.pp /tmp +semodule -i /tmp/dockercvmfs.pp + +# install kubernetes volume plugin +mkdir -p /var/lib/kubelet/plugins/volume/cern~cvmfs +docker cp docker-volume-cvmfs:/usr/sbin/docker-volume-cvmfs /var/lib/kubelet/plugins/volume/cern~cvmfs/cvmfs + +# TODO: move this elsewhere +lvextend /dev/atomicos/root --size 5G +xfs_growfs /dev/mapper/atomicos-root diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 745709189..f51f5101a 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -53,7 +53,7 @@ sed -i ' # the option --hostname-override for kubelet uses the hostname to register the node. # Using any other name will break the load balancer and cinder volume features. HOSTNAME=$(hostname -I | cut -d' ' -f1) -KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 --hostname-override=${HOSTNAME} --cluster-dns=10.254.10.10 --cluster-domain=cluster.local ${KUBE_CONFIG}" +KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 --hostname-override=${HOSTNAME} --cluster-dns=10.254.10.10 --cluster-domain=cluster.local --volume-plugin-dir /var/lib/kubelet/plugins/volume/exec ${KUBE_CONFIG}" if [ -n "${INSECURE_REGISTRY_URL}" ]; then KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml index d455a23e4..efa1817f4 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml @@ -43,3 +43,5 @@ write_files: TRUST_ID="$TRUST_ID" AUTH_URL="$AUTH_URL" INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL" + CVMFS_ENABLED="$CVMFS_ENABLED" + CVMFS_TAG="$CVMFS_TAG" diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 692d9b093..add4a8c8a 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -263,6 +263,18 @@ parameters: description: insecure registry url default: "" + cvmfs_enabled: + type: boolean + description: > + Indicates whether cvmfs support should be enabled in the cluster. + default: true + + cvmfs_tag: + type: string + description: > + The tag to use for the docker-volume-cvmfs container. + default: "latest" + resources: ###################################################################### @@ -373,6 +385,8 @@ resources: trust_id: {get_param: trust_id} auth_url: {get_param: auth_url} insecure_registry_url: {get_param: insecure_registry_url} + cvmfs_enabled: {get_param: cvmfs_enabled} + cvmfs_tag: {get_param: cvmfs_tag} outputs: diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index 4811e833a..024a002a5 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -193,6 +193,18 @@ parameters: insecure_registry_url: type: string description: insecure registry url + + cvmfs_enabled: + type: boolean + description: > + Indicates whether cvmfs support should be enabled in the cluster. + default: true + + cvmfs_tag: + type: string + description: > + The tag to use for the docker-volume-cvmfs container. + default: "latest" resources: @@ -256,6 +268,8 @@ resources: $TRUST_ID: {get_param: trust_id} $AUTH_URL: {get_param: auth_url} $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url} + $CVMFS_ENABLED: {get_param: cvmfs_enabled} + $CVMFS_TAG: {get_param: cvmfs_tag} write_kubeconfig: type: OS::Heat::SoftwareConfig @@ -356,6 +370,12 @@ resources: group: ungrouped config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh} + configure_cvmfs: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: ../../common/templates/fragments/configure-cvmfs.sh} + kube_minion_init: type: OS::Heat::MultipartMime properties: @@ -374,6 +394,7 @@ resources: - config: {get_resource: enable_kube_proxy} - config: {get_resource: enable_docker_registry} - config: {get_resource: install_cern_certs} + - config: {get_resource: configure_cvmfs} - config: {get_resource: minion_wc_notify} ###################################################################### diff --git a/magnum/drivers/swarm_fedora_atomic_v1/template_def.py b/magnum/drivers/swarm_fedora_atomic_v1/template_def.py index 103c7db50..e76230418 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/template_def.py +++ b/magnum/drivers/swarm_fedora_atomic_v1/template_def.py @@ -100,7 +100,8 @@ class AtomicSwarmTemplateDefinition(template_def.BaseTemplateDefinition): extra_params['magnum_url'] = osc.magnum_url() label_list = ['flannel_network_cidr', 'flannel_backend', - 'flannel_network_subnetlen', 'rexray_preempt'] + 'flannel_network_subnetlen', 'rexray_preempt', + 'cvmfs_enabled', 'cvmfs_tag'] extra_params['auth_url'] = context.auth_url diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml index 3d1c75be3..99309b3ca 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml @@ -226,6 +226,18 @@ parameters: other hosts are using the volume default: "false" + cvmfs_enabled: + type: boolean + description: > + Indicates whether cvmfs support should be enabled in the cluster. + default: true + + cvmfs_tag: + type: string + description: > + The tag to use for the docker-volume-cvmfs container. + default: "latest" + resources: @@ -321,6 +333,8 @@ resources: registry_chunksize: {get_param: registry_chunksize} volume_driver: {get_param: volume_driver} rexray_preempt: {get_param: rexray_preempt} + cvmfs_enabled: {get_param: cvmfs_enabled} + cvmfs_tag: {get_param: cvmfs_tag} outputs: diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.sh index 872ed7ea6..6cd13bde1 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.sh +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.sh @@ -36,4 +36,6 @@ REGISTRY_INSECURE="$REGISTRY_INSECURE" REGISTRY_CHUNKSIZE="$REGISTRY_CHUNKSIZE" VOLUME_DRIVER="$VOLUME_DRIVER" REXRAY_PREEMPT="$REXRAY_PREEMPT" +CVMFS_ENABLED="$CVMFS_ENABLED" +CVMFS_TAG="$CVMFS_TAG" END_SERVICE diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index 507fecb7b..f9868f4a7 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -161,6 +161,18 @@ parameters: other hosts are using the volume default: "false" + cvmfs_enabled: + type: boolean + description: > + Indicates whether cvmfs support should be enabled in the cluster. + default: true + + cvmfs_tag: + type: string + description: > + The tag to use for the docker-volume-cvmfs container. + default: "latest" + resources: node_wait_handle: @@ -217,6 +229,8 @@ resources: "$REGISTRY_CHUNKSIZE": {get_param: registry_chunksize} "$VOLUME_DRIVER": {get_param: volume_driver} "$REXRAY_PREEMPT": {get_param: rexray_preempt} + "$CVMFS_ENABLED": {get_param: cvmfs_enabled} + "$CVMFS_TAG": {get_param: cvmfs_tag} remove_docker_key: type: "OS::Heat::SoftwareConfig" @@ -328,6 +342,12 @@ resources: group: ungrouped config: {get_file: ../../common/templates/fragments/install-cern-certs.sh} + configure_docker_cvmfs: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: ../../common/templates/fragments/configure-cvmfs.sh} + swarm_node_init: type: "OS::Heat::MultipartMime" properties: @@ -347,8 +367,9 @@ resources: - config: {get_resource: enable_docker_registry} - config: {get_resource: enable_services} - config: {get_resource: install_cern_certs} - - config: {get_resource: cfn_signal} - config: {get_resource: volume_service} + - config: {get_resource: configure_docker_cvmfs} + - config: {get_resource: cfn_signal} swarm-node: type: "OS::Nova::Server" -- GitLab