From 693e46c260ebff24d804a682ac29e4f30c13ea93 Mon Sep 17 00:00:00 2001 From: Jaycen Grant Date: Mon, 3 Oct 2016 13:54:11 -0700 Subject: [PATCH 1/9] [cern] Allow keypair to be added during cluster create Cherry-Pick: https://review.openstack.org/#/c/381307/ Keypair can now be added during Cluster create command. This allows Clusters to be created from the same ClusterTemplate but have different keypair values when created. If not specified on create then the value from the ClusterTemplate will be used. Adds keypair_id to Cluster object and uses of keypair_id will use the value from Cluster instead of ClusterTemplate. Change-Id: I177a5aa06f881156944a9f74c9ccc3cd2abac492 Implements: blueprint keypair-override-on-create --- doc/source/userguide.rst | 15 +++++++-- magnum/api/attr_validator.py | 6 ++-- magnum/api/controllers/v1/bay.py | 9 ++++-- magnum/api/controllers/v1/cluster.py | 14 ++++++-- magnum/api/controllers/v1/cluster_template.py | 2 +- .../bc46ba6cf949_add_keypair_to_cluster.py | 32 +++++++++++++++++++ magnum/db/sqlalchemy/models.py | 1 + magnum/drivers/common/template_def.py | 2 +- magnum/objects/cluster.py | 4 ++- .../tests/functional/api/v1/test_baymodel.py | 7 ---- .../api/v1/test_cluster_template.py | 6 ++-- magnum/tests/functional/common/datagen.py | 2 +- .../unit/api/controllers/v1/test_cluster.py | 18 +++++++++++ .../controllers/v1/test_cluster_template.py | 13 +++++--- magnum/tests/unit/api/test_attr_validator.py | 15 +++++++++ .../handlers/test_k8s_cluster_conductor.py | 1 + .../handlers/test_mesos_cluster_conductor.py | 1 + .../handlers/test_swarm_cluster_conductor.py | 1 + magnum/tests/unit/objects/test_cluster.py | 1 + magnum/tests/unit/objects/test_objects.py | 2 +- 20 files changed, 122 insertions(+), 30 deletions(-) create mode 100644 magnum/db/sqlalchemy/alembic/versions/bc46ba6cf949_add_keypair_to_cluster.py diff --git a/doc/source/userguide.rst b/doc/source/userguide.rst index 7996c4184..658e8c84a 100644 --- a/doc/source/userguide.rst +++ b/doc/source/userguide.rst @@ -101,13 +101,15 @@ They are loosely grouped as: mandatory, infrastructure, COE specific. Mesos Ubuntu ========== ===================== -This is a mandatory parameter and there is no default value. + This is a mandatory parameter and there is no default value. --keypair-id \ The name or UUID of the SSH keypair to configure in the cluster servers for ssh access. You will need the key to be able to ssh to the servers in the cluster. The login name is specific to the cluster - driver. This is a mandatory parameter and there is no default value. + driver. If keypair is not provided in template it will be required at + Cluster create. This value will be overridden by any keypair value that + is provided during Cluster create. --external-network-id \ The name or network ID of a Neutron network to provide connectivity @@ -427,6 +429,15 @@ follows: name will be generated using a string and a number, for example "gamma-7-cluster". +--keypair \ + The name or UUID of the SSH keypair to configure in the cluster servers + for ssh access. You will need the key to be able to ssh to the + servers in the cluster. The login name is specific to the cluster + driver. If keypair is not provided it will attempt to use the value in + the ClusterTemplate. If the ClusterTemplate is also missing a keypair value + then an error will be returned. The keypair value provided here will + override the keypair value from the ClusterTemplate. + --node-count \ The number of servers that will serve as node in the cluster. The default is 1. diff --git a/magnum/api/attr_validator.py b/magnum/api/attr_validator.py index bcd7999d0..537bf76c5 100644 --- a/magnum/api/attr_validator.py +++ b/magnum/api/attr_validator.py @@ -166,7 +166,7 @@ def validate_labels_executor_env_variables(labels): raise exception.InvalidParameterValue(err) -def validate_os_resources(context, cluster_template): +def validate_os_resources(context, cluster_template, cluster=None): """Validate ClusterTemplate's OpenStack Resources""" cli = clients.OpenStackClients(context) @@ -178,6 +178,9 @@ def validate_os_resources(context, cluster_template): else: validate_method(cluster_template[attr]) + if cluster: + validate_keypair(cli, cluster['keypair']) + def validate_master_count(cluster, cluster_template): if cluster['master_count'] > 1 and \ @@ -190,7 +193,6 @@ def validate_master_count(cluster, cluster_template): validators = {'image_id': validate_image, 'flavor_id': validate_flavor, 'master_flavor_id': validate_flavor, - 'keypair_id': validate_keypair, 'external_network_id': validate_external_network, 'fixed_network': validate_fixed_network, 'labels': validate_labels} diff --git a/magnum/api/controllers/v1/bay.py b/magnum/api/controllers/v1/bay.py index a7ac8c094..a8faecd48 100644 --- a/magnum/api/controllers/v1/bay.py +++ b/magnum/api/controllers/v1/bay.py @@ -415,9 +415,13 @@ class BaysController(base.Controller): action='bay:create') baymodel = objects.ClusterTemplate.get_by_uuid(context, bay.baymodel_id) - attr_validator.validate_os_resources(context, baymodel.as_dict()) - attr_validator.validate_master_count(bay.as_dict(), baymodel.as_dict()) + bay_dict = bay.as_dict() + bay_dict['keypair'] = baymodel.keypair_id + attr_validator.validate_os_resources(context, baymodel.as_dict(), + bay_dict) + attr_validator.validate_master_count(bay.as_dict(), baymodel.as_dict()) + bay_dict['project_id'] = context.project_id bay_dict['user_id'] = context.user_id # NOTE(yuywz): We will generate a random human-readable name for @@ -426,7 +430,6 @@ class BaysController(base.Controller): bay_dict['name'] = name bay_dict['coe_version'] = None bay_dict['container_version'] = None - new_bay = objects.Cluster(context, **bay_dict) new_bay.uuid = uuid.uuid4() return new_bay diff --git a/magnum/api/controllers/v1/cluster.py b/magnum/api/controllers/v1/cluster.py index a4fe83f6a..a7d9c9f23 100644 --- a/magnum/api/controllers/v1/cluster.py +++ b/magnum/api/controllers/v1/cluster.py @@ -95,6 +95,10 @@ class Cluster(base.APIBase): mandatory=True) """The cluster_template UUID""" + keypair = wsme.wsattr(wtypes.StringType(min_length=1, max_length=255), + default=None) + """The name or id of the nova ssh keypair""" + node_count = wsme.wsattr(wtypes.IntegerType(minimum=1), default=1) """The node count for this cluster. Default to 1 if not set""" @@ -152,7 +156,7 @@ class Cluster(base.APIBase): def _convert_with_links(cluster, url, expand=True): if not expand: cluster.unset_fields_except(['uuid', 'name', 'cluster_template_id', - 'node_count', 'status', + 'keypair', 'node_count', 'status', 'create_timeout', 'master_count', 'stack_id']) @@ -174,6 +178,7 @@ class Cluster(base.APIBase): sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c', name='example', cluster_template_id=temp_id, + keypair=None, node_count=2, master_count=1, create_timeout=15, @@ -360,10 +365,15 @@ class ClustersController(base.Controller): temp_id = cluster.cluster_template_id cluster_template = objects.ClusterTemplate.get_by_uuid(context, temp_id) + # If keypair not present, use cluster_template value + if cluster.keypair is None: + cluster.keypair = cluster_template.keypair_id + cluster_dict = cluster.as_dict() attr_validator.validate_os_resources(context, - cluster_template.as_dict()) + cluster_template.as_dict(), + cluster_dict) attr_validator.validate_master_count(cluster_dict, cluster_template.as_dict()) diff --git a/magnum/api/controllers/v1/cluster_template.py b/magnum/api/controllers/v1/cluster_template.py index bd0325d31..f564f9ee7 100644 --- a/magnum/api/controllers/v1/cluster_template.py +++ b/magnum/api/controllers/v1/cluster_template.py @@ -64,7 +64,7 @@ class ClusterTemplate(base.APIBase): """The DNS nameserver address""" keypair_id = wsme.wsattr(wtypes.StringType(min_length=1, max_length=255), - mandatory=True) + default=None) """The name or id of the nova ssh keypair""" external_network_id = wtypes.StringType(min_length=1, max_length=255) diff --git a/magnum/db/sqlalchemy/alembic/versions/bc46ba6cf949_add_keypair_to_cluster.py b/magnum/db/sqlalchemy/alembic/versions/bc46ba6cf949_add_keypair_to_cluster.py new file mode 100644 index 000000000..d40c265fc --- /dev/null +++ b/magnum/db/sqlalchemy/alembic/versions/bc46ba6cf949_add_keypair_to_cluster.py @@ -0,0 +1,32 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""add keypair to cluster + +Revision ID: bc46ba6cf949 +Revises: 720f640f43d1 +Create Date: 2016-10-03 10:47:08.584635 + +""" + +# revision identifiers, used by Alembic. +revision = 'bc46ba6cf949' +down_revision = '720f640f43d1' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('cluster', sa.Column('keypair', sa.String(length=255), + nullable=True)) diff --git a/magnum/db/sqlalchemy/models.py b/magnum/db/sqlalchemy/models.py index 621fa0ab5..0009e7b4b 100644 --- a/magnum/db/sqlalchemy/models.py +++ b/magnum/db/sqlalchemy/models.py @@ -113,6 +113,7 @@ class Cluster(Base): uuid = Column(String(36)) name = Column(String(255)) cluster_template_id = Column(String(255)) + keypair = Column(String(255)) stack_id = Column(String(255)) api_address = Column(String(255)) node_addresses = Column(JSONEncodedList) diff --git a/magnum/drivers/common/template_def.py b/magnum/drivers/common/template_def.py index d956fbaa3..acf125d63 100644 --- a/magnum/drivers/common/template_def.py +++ b/magnum/drivers/common/template_def.py @@ -344,7 +344,7 @@ class BaseTemplateDefinition(TemplateDefinition): self._osc = None self.add_parameter('ssh_key_name', - cluster_template_attr='keypair_id', + cluster_attr='keypair', required=True) self.add_parameter('server_image', cluster_template_attr='image_id') diff --git a/magnum/objects/cluster.py b/magnum/objects/cluster.py index 33c4d9f8b..20975acd7 100644 --- a/magnum/objects/cluster.py +++ b/magnum/objects/cluster.py @@ -41,8 +41,9 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject, # Version 1.9: Rename table name from 'bay' to 'cluster' # Rename 'baymodel_id' to 'cluster_template_id' # Rename 'bay_create_timeout' to 'create_timeout' + # Version 1.10: Added 'keypair' field - VERSION = '1.9' + VERSION = '1.10' dbapi = dbapi.get_instance() @@ -53,6 +54,7 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject, 'project_id': fields.StringField(nullable=True), 'user_id': fields.StringField(nullable=True), 'cluster_template_id': fields.StringField(nullable=True), + 'keypair': fields.StringField(nullable=True), 'stack_id': fields.StringField(nullable=True), 'status': m_fields.ClusterStatusField(nullable=True), 'status_reason': fields.StringField(nullable=True), diff --git a/magnum/tests/functional/api/v1/test_baymodel.py b/magnum/tests/functional/api/v1/test_baymodel.py index b2f732a54..5dfaef994 100644 --- a/magnum/tests/functional/api/v1/test_baymodel.py +++ b/magnum/tests/functional/api/v1/test_baymodel.py @@ -185,13 +185,6 @@ class BayModelTest(base.BaseTempestTest): exceptions.BadRequest, self.baymodel_client.post_baymodel, gen_model) - @testtools.testcase.attr('negative') - def test_create_baymodel_missing_keypair(self): - gen_model = datagen.baymodel_data_with_missing_keypair() - self.assertRaises( - exceptions.NotFound, - self.baymodel_client.post_baymodel, gen_model) - @testtools.testcase.attr('negative') def test_update_baymodel_invalid_patch(self): # get json object diff --git a/magnum/tests/functional/api/v1/test_cluster_template.py b/magnum/tests/functional/api/v1/test_cluster_template.py index 79b5a922a..29b8a441f 100644 --- a/magnum/tests/functional/api/v1/test_cluster_template.py +++ b/magnum/tests/functional/api/v1/test_cluster_template.py @@ -200,13 +200,11 @@ class ClusterTemplateTest(base.BaseTempestTest): exceptions.BadRequest, self.cluster_template_client.post_cluster_template, gen_model) - @testtools.testcase.attr('negative') + @testtools.testcase.attr('positive') def test_create_cluster_template_missing_keypair(self): gen_model = \ datagen.cluster_template_data_with_missing_keypair() - self.assertRaises( - exceptions.NotFound, - self.cluster_template_client.post_cluster_template, gen_model) + resp, model = self._create_cluster_template(gen_model) @testtools.testcase.attr('negative') def test_update_cluster_template_invalid_patch(self): diff --git a/magnum/tests/functional/common/datagen.py b/magnum/tests/functional/common/datagen.py index c92cbf51a..c987cfb1a 100644 --- a/magnum/tests/functional/common/datagen.py +++ b/magnum/tests/functional/common/datagen.py @@ -502,7 +502,6 @@ def valid_swarm_cluster_template(is_public=False): public=is_public, dns_nameserver=config.Config.dns_nameserver, master_flavor_id=master_flavor_id, - keypair_id=config.Config.keypair_id, coe="swarm", docker_volume_size=3, cluster_distro=None, external_network_id=config.Config.nic_id, @@ -535,6 +534,7 @@ def cluster_data(name=data_utils.rand_name('cluster'), data = { "name": name, "cluster_template_id": cluster_template_id, + "keypair": config.Config.keypair_id, "node_count": node_count, "discovery_url": None, "create_timeout": create_timeout, diff --git a/magnum/tests/unit/api/controllers/v1/test_cluster.py b/magnum/tests/unit/api/controllers/v1/test_cluster.py index 6ec796836..9eb84188c 100644 --- a/magnum/tests/unit/api/controllers/v1/test_cluster.py +++ b/magnum/tests/unit/api/controllers/v1/test_cluster.py @@ -741,6 +741,24 @@ class TestPost(api_base.FunctionalTest): self.assertEqual('application/json', response.content_type) self.assertEqual(400, response.status_int) + def test_create_cluster_with_keypair(self): + bdict = apiutils.cluster_post_data() + bdict['keypair'] = 'keypair2' + response = self.post_json('/clusters', bdict) + self.assertEqual('application/json', response.content_type) + self.assertEqual(202, response.status_int) + cluster, timeout = self.mock_cluster_create.call_args + self.assertEqual('keypair2', cluster[0].keypair) + + def test_create_cluster_without_keypair(self): + bdict = apiutils.cluster_post_data() + response = self.post_json('/clusters', bdict) + self.assertEqual('application/json', response.content_type) + self.assertEqual(202, response.status_int) + cluster, timeout = self.mock_cluster_create.call_args + # Verify keypair from ClusterTemplate is used + self.assertEqual('keypair1', cluster[0].keypair) + class TestDelete(api_base.FunctionalTest): def setUp(self): diff --git a/magnum/tests/unit/api/controllers/v1/test_cluster_template.py b/magnum/tests/unit/api/controllers/v1/test_cluster_template.py index 27963996e..277e64057 100644 --- a/magnum/tests/unit/api/controllers/v1/test_cluster_template.py +++ b/magnum/tests/unit/api/controllers/v1/test_cluster_template.py @@ -442,7 +442,7 @@ class TestPatch(api_base.FunctionalTest): self.assertTrue(response.json['errors']) def test_remove_mandatory_property_fail(self): - mandatory_properties = ('/image_id', '/keypair_id', '/coe', + mandatory_properties = ('/image_id', '/coe', '/external_network_id', '/server_type', '/tls_disabled', '/public', '/registry_enabled', @@ -860,12 +860,15 @@ class TestPost(api_base.FunctionalTest): expect_errors=True) self.assertEqual(400, response.status_int) - def test_create_cluster_template_without_keypair_id(self): + @mock.patch('magnum.api.attr_validator.validate_image') + def test_create_cluster_template_without_keypair_id(self, + mock_image_data): + mock_image_data.return_value = {'name': 'mock_name', + 'os_distro': 'fedora-atomic'} bdict = apiutils.cluster_template_post_data() del bdict['keypair_id'] - response = self.post_json('/clustertemplates', bdict, - expect_errors=True) - self.assertEqual(400, response.status_int) + response = self.post_json('/clustertemplates', bdict) + self.assertEqual(201, response.status_int) @mock.patch('magnum.api.attr_validator.validate_image') def test_create_cluster_template_with_dns(self, diff --git a/magnum/tests/unit/api/test_attr_validator.py b/magnum/tests/unit/api/test_attr_validator.py index 007ea3f3d..04d7778f3 100644 --- a/magnum/tests/unit/api/test_attr_validator.py +++ b/magnum/tests/unit/api/test_attr_validator.py @@ -297,3 +297,18 @@ class TestAttrValidator(base.BaseTestCase): mock_context = mock.MagicMock() attr_validator.validate_os_resources(mock_context, mock_cluster_template) + + @mock.patch('magnum.common.clients.OpenStackClients') + def test_validate_os_resources_with_cluster(self, mock_os_cli): + mock_cluster_template = {} + mock_cluster = {'keypair': 'test-keypair'} + mock_keypair = mock.MagicMock() + mock_keypair.id = 'test-keypair' + mock_nova = mock.MagicMock() + mock_nova.keypairs.get.return_value = mock_keypair + mock_os_cli = mock.MagicMock() + mock_os_cli.nova.return_value = mock_nova + mock_context = mock.MagicMock() + attr_validator.validate_os_resources(mock_context, + mock_cluster_template, + mock_cluster) diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index 6bf00f759..6da8e836e 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -54,6 +54,7 @@ class TestClusterConductorWithK8s(base.TestCase): self.cluster_dict = { 'uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', 'cluster_template_id': 'xx-xx-xx-xx', + 'keypair': 'keypair_id', 'name': 'cluster1', 'stack_id': 'xx-xx-xx-xx', 'api_address': '172.17.2.3', diff --git a/magnum/tests/unit/conductor/handlers/test_mesos_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_mesos_cluster_conductor.py index f69af100d..0fd98d0a7 100644 --- a/magnum/tests/unit/conductor/handlers/test_mesos_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_mesos_cluster_conductor.py @@ -52,6 +52,7 @@ class TestClusterConductorWithMesos(base.TestCase): 'id': 1, 'uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', 'cluster_template_id': 'xx-xx-xx-xx', + 'keypair': 'keypair_id', 'name': 'cluster1', 'stack_id': 'xx-xx-xx-xx', 'api_address': '172.17.2.3', diff --git a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py index d1acf3cc1..9301309f1 100644 --- a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py @@ -55,6 +55,7 @@ class TestClusterConductorWithSwarm(base.TestCase): 'id': 1, 'uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', 'cluster_template_id': 'xx-xx-xx-xx', + 'keypair': 'keypair_id', 'name': 'cluster1', 'stack_id': 'xx-xx-xx-xx', 'api_address': '172.17.2.3', diff --git a/magnum/tests/unit/objects/test_cluster.py b/magnum/tests/unit/objects/test_cluster.py index 0b62700e7..c13b8bd2f 100644 --- a/magnum/tests/unit/objects/test_cluster.py +++ b/magnum/tests/unit/objects/test_cluster.py @@ -37,6 +37,7 @@ class TestClusterObject(base.DbTestCase): cluster_template_id = self.fake_cluster['cluster_template_id'] self.fake_cluster_template = objects.ClusterTemplate( uuid=cluster_template_id) + self.fake_cluster['keypair'] = 'keypair1' @mock.patch('magnum.objects.ClusterTemplate.get_by_uuid') def test_get_by_id(self, mock_cluster_template_get): diff --git a/magnum/tests/unit/objects/test_objects.py b/magnum/tests/unit/objects/test_objects.py index 5d689ec31..cea5e81e9 100644 --- a/magnum/tests/unit/objects/test_objects.py +++ b/magnum/tests/unit/objects/test_objects.py @@ -362,7 +362,7 @@ class TestObject(test_base.TestCase, _TestObject): # For more information on object version testing, read # http://docs.openstack.org/developer/magnum/objects.html object_data = { - 'Cluster': '1.9-f9838e23eef5f1a7d9606c1ccce21800', + 'Cluster': '1.10-377082b6d7895cd800a39fa004765538', 'ClusterTemplate': '1.17-65a95ef932dd08800a83871eb3cf312b', 'Certificate': '1.1-1924dc077daa844f0f9076332ef96815', 'MyObj': '1.0-b43567e512438205e32f4e95ca616697', -- GitLab From 7ee9beabc7688d3b2326d700346fddc11a9dd4a6 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Thu, 3 Nov 2016 15:02:04 +0100 Subject: [PATCH 2/9] [cern] disable neutron check on external network allow magnum usage with nova network by disabling the requirement for neutron. this won't go upstream as it's not generic enough, but can be dropped once we finish migrating all cells to neutron. Change-Id: Ie98beae6b97cd2cb7f43aaf66d4e03aa7dac79b1 --- magnum/api/attr_validator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/magnum/api/attr_validator.py b/magnum/api/attr_validator.py index 537bf76c5..2b67511ac 100644 --- a/magnum/api/attr_validator.py +++ b/magnum/api/attr_validator.py @@ -72,6 +72,9 @@ def validate_keypair(cli, keypair): def validate_external_network(cli, external_network): """Validate external network""" + # TODO(mvelten): hack to get this working with nova network + # replace with upstream solution later + return count = 0 ext_filter = {'router:external': True} -- GitLab From 0685135160d61b45a613fa80911f5e7c33814d67 Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Fri, 30 Sep 2016 15:10:52 +0200 Subject: [PATCH 3/9] [cern] Make cinder volume optional Cherry-Pick: https://review.openstack.org/#/c/391830/ In the swarm_atomic and k8s_atomic drivers container images are stored in a dedicated cinder volume per cluster node. It is proven that this architecture can be a scalability bottleneck. Make the use of cinder volumes for container images and opt-in option. If docker-volume-size is not specified no cinder volumes will be created. Before, if docker-volume-size wasn't specified the default value was 25. To use cinder volumes for container storage the user will interact with magnum as before, (meaning the valid values are integers starting from 1). Closes-Bug: #1638006 Change-Id: I3394c62a43bbf950b7cf0b86a71b1d9b0481d68f Conflicts: * magnum/drivers/common/swarm_fedora_template_def.py [edited magnum/drivers/swarm_fedora_atomic_v1/template_def.py instead ] * magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py Removed the unit test for the empty docker_volume_size case. Fixed the new unit to not metnion the driver. --- doc/source/userguide.rst | 80 ++++++++--------- .../drivers/common/k8s_fedora_template_def.py | 20 +++-- .../templates/environments/no_volume.yaml | 4 + .../templates/environments/with_volume.yaml | 4 + .../fragments/configure-docker-storage.sh | 46 +++++----- .../configure_docker_storage_driver_atomic.sh | 16 ++-- .../fragments/write-heat-params-master.yaml | 1 + .../fragments/write-heat-params.yaml | 1 + .../templates/kubecluster.yaml | 2 +- .../templates/kubemaster.yaml | 5 +- .../templates/kubeminion.yaml | 5 +- .../templates/kubecluster.yaml | 4 +- .../templates/kubemaster.yaml | 7 ++ .../templates/kubeminion.yaml | 7 ++ .../swarm_fedora_atomic_v1/template_def.py | 13 ++- .../templates/cluster.yaml | 2 +- .../fragments/write-heat-params-master.yaml | 1 + .../fragments/write-heat-params-node.yaml | 1 + .../templates/swarmmaster.yaml | 5 +- .../templates/swarmnode.yaml | 5 +- .../unit/api/controllers/v1/test_baymodel.py | 1 - .../controllers/v1/test_cluster_template.py | 1 - .../handlers/test_k8s_cluster_conductor.py | 88 ++++++++++++++++--- .../handlers/test_swarm_cluster_conductor.py | 15 ++-- .../no-cinder-volume-87b9339e066c30a0.yaml | 10 +++ 25 files changed, 230 insertions(+), 114 deletions(-) create mode 100644 magnum/drivers/common/templates/environments/no_volume.yaml create mode 100644 magnum/drivers/common/templates/environments/with_volume.yaml create mode 100644 releasenotes/notes/no-cinder-volume-87b9339e066c30a0.yaml diff --git a/doc/source/userguide.rst b/doc/source/userguide.rst index 658e8c84a..9dd0cab50 100644 --- a/doc/source/userguide.rst +++ b/doc/source/userguide.rst @@ -206,11 +206,11 @@ They are loosely grouped as: mandatory, infrastructure, COE specific. is 'None'. --docker-volume-size \ - The size in GB for the local storage on each server for the Docker - daemon to cache the images and host the containers. Cinder volumes - provide the storage. The default is 25 GB. For the 'devicemapper' - storage driver, the minimum value is 3GB. For the 'overlay' storage - driver, the minimum value is 1GB. + If specified, container images will be stored in a cinder volume of the + specified size in GB. Each cluster node will have a volume attached of + the above size. If not specified, images will be stored in the compute + instance's local disk. For the 'devicemapper' storage driver, the minimum + value is 3GB. For the 'overlay' storage driver, the minimum value is 1GB. --docker-storage-driver \ The name of a driver to manage the storage for the images and the @@ -355,8 +355,8 @@ Network needed. Storage - Cinder provides the block storage that is used for both hosting the - containers as well as persistent storage for the containers. + Cinder provides the block storage that can be used to host the + containers and as persistent storage for the containers. Security Barbican provides the storage of secrets such as certificates used @@ -868,14 +868,8 @@ Volume driver (volume-driver) Storage driver (docker-storage-driver) Specified in the ClusterTemplate to select the Docker storage driver. The supported storage drivers are 'devicemapper' and 'overlay', with - 'devicemapper' being the default. You may get better performance with - the overlay driver depending on your use patterns, with the requirement - that SELinux must be disabled inside the containers, although it still runs - in enforcing mode on the cluster servers. Magnum will create a Cinder volume - for each node, mount it on the node and configure it as a logical - volume named 'docker'. The Docker daemon will run the selected device - driver to manage this logical volume and host the container writable - layer there. Refer to the `Storage`_ section for more details. + 'devicemapper' being the default. Refer to the `Storage`_ section for more + details. Image (image-id) Specified in the ClusterTemplate to indicate the image to boot the servers. @@ -1013,15 +1007,8 @@ Volume driver (volume-driver) Storage driver (docker-storage-driver) Specified in the ClusterTemplate to select the Docker storage driver. The supported storage driver are 'devicemapper' and 'overlay', with - 'devicemapper' being the default. You may get better performance with - the 'overlay' driver depending on your use patterns, with the requirement - that SELinux must be disabled inside the containers, although it still runs - in enforcing mode on the cluster servers. Magnum will create a Cinder volume - for each node and attach it as a device. Then depending on the driver, - additional configuration is performed to make the volume available to - the particular driver. For instance, 'devicemapper' uses LVM; therefore - Magnum will create physical volume and logical volume using the attached - device. Refer to the `Storage`_ section for more details. + 'devicemapper' being the default. Refer to the `Storage`_ section for more + details. Image (image-id) Specified in the ClusterTemplate to indicate the image to boot the servers @@ -1996,25 +1983,32 @@ configured in the Docker daemon through a number of storage options. When the container is removed, the storage allocated to the particular container is also deleted. -To manage this space in a flexible manner independent of the Nova -instance flavor, Magnum creates a separate Cinder block volume for each -node in the cluster, mounts it to the node and configures it to be used as -ephemeral storage. Users can specify the size of the Cinder volume with -the ClusterTemplate attribute 'docker-volume-size'. The default size is 5GB. -Currently the block size is fixed at cluster creation time, but future -lifecycle operations may allow modifying the block size during the -life of the cluster. - -To use the Cinder block storage, there is a number of Docker -storage drivers available. Only 'devicemapper' is supported as the -storage driver but other drivers such as 'OverlayFS' are being -considered. There are important trade-off between the choices -for the storage drivers that should be considered. For instance, -'OperlayFS' may offer better performance, but it may not support -the filesystem metadata needed to use SELinux, which is required -to support strong isolation between containers running in the same -cluster. Using the 'devicemapper' driver does allow the use of SELinux. - +Magnum can manage the containers' filesystem in two ways, storing them +on the local disk of the compute instances or in a separate Cinder block +volume for each node in the cluster, mounts it to the node and +configures it to be used as ephemeral storage. Users can specify the +size of the Cinder volume with the ClusterTemplate attribute +'docker-volume-size'. Currently the block size is fixed at cluster +creation time, but future lifecycle operations may allow modifying the +block size during the life of the cluster. + +Both local disk and the Cinder block storage can be used with a number +of Docker storage drivers available. + +* 'devicemapper': When used with a dedicated Cinder volume it is + configured using direct-lvm and offers very good performance. If it's + used with the compute instance's local disk uses a loopback device + offering poor performance and it's not recommended for production + environments. Using the 'devicemapper' driver does allow the use of + SELinux. + +* 'overlay' When used with a dedicated Cinder volume offers as good + or better performance than devicemapper. If used on the local disk of + the compute instance (especially with high IOPS drives) you can get + significant performance gains. However, for kernel versions less than + 4.9, SELinux must be disabled inside the containers resulting in worse + container isolation, although it still runs in enforcing mode on the + cluster compute instances. Persistent storage ------------------ diff --git a/magnum/drivers/common/k8s_fedora_template_def.py b/magnum/drivers/common/k8s_fedora_template_def.py index 67d467133..b7cc9c1c0 100644 --- a/magnum/drivers/common/k8s_fedora_template_def.py +++ b/magnum/drivers/common/k8s_fedora_template_def.py @@ -80,17 +80,19 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): def get_env_files(self, cluster_template): env_files = [] + + if cluster_template.docker_volume_size is None: + env_files.append('no_volume.yaml') + else: + env_files.append('with_volume.yaml') + if cluster_template.master_lb_enabled: - env_files.append( - template_def.COMMON_ENV_PATH + 'with_master_lb.yaml') + env_files.append('with_master_lb.yaml') else: - env_files.append( - template_def.COMMON_ENV_PATH + 'no_master_lb.yaml') + env_files.append('no_master_lb.yaml') if cluster_template.floating_ip_enabled: - env_files.append( - template_def.COMMON_ENV_PATH + 'enable_floating_ip.yaml') + env_files.append('enable_floating_ip.yaml') else: - env_files.append( - template_def.COMMON_ENV_PATH + 'disable_floating_ip.yaml') + env_files.append('disable_floating_ip.yaml') - return env_files + return [template_def.COMMON_ENV_PATH + ef for ef in env_files] diff --git a/magnum/drivers/common/templates/environments/no_volume.yaml b/magnum/drivers/common/templates/environments/no_volume.yaml new file mode 100644 index 000000000..8e2dc31ad --- /dev/null +++ b/magnum/drivers/common/templates/environments/no_volume.yaml @@ -0,0 +1,4 @@ +# Environment file to NOT use a cinder volume to store containers +resource_registry: + "Magnum::Optional::Cinder::Volume": "OS::Heat::None" + "Magnum::Optional::Cinder::VolumeAttachment": "OS::Heat::None" diff --git a/magnum/drivers/common/templates/environments/with_volume.yaml b/magnum/drivers/common/templates/environments/with_volume.yaml new file mode 100644 index 000000000..e67f28944 --- /dev/null +++ b/magnum/drivers/common/templates/environments/with_volume.yaml @@ -0,0 +1,4 @@ +# Environment file to use a cinder volume to store containers +resource_registry: + "Magnum::Optional::Cinder::Volume": "OS::Cinder::Volume" + "Magnum::Optional::Cinder::VolumeAttachment": "OS::Cinder::VolumeAttachment" diff --git a/magnum/drivers/common/templates/fragments/configure-docker-storage.sh b/magnum/drivers/common/templates/fragments/configure-docker-storage.sh index e52dbc505..104c3ac5e 100644 --- a/magnum/drivers/common/templates/fragments/configure-docker-storage.sh +++ b/magnum/drivers/common/templates/fragments/configure-docker-storage.sh @@ -2,30 +2,32 @@ . /etc/sysconfig/heat-params -if [ "$ENABLE_CINDER" == "False" ]; then - # FIXME(yuanying): Use ephemeral disk for docker storage - # Currently Ironic doesn't support cinder volumes, - # so we must use preserved ephemeral disk instead of a cinder volume. - device_path=$(readlink -f /dev/disk/by-label/ephemeral0) -else - attempts=60 - while [ ${attempts} -gt 0 ]; do - device_name=$(ls /dev/disk/by-id | grep ${DOCKER_VOLUME:0:20}$) - if [ -n "${device_name}" ]; then - break - fi - echo "waiting for disk device" - sleep 0.5 - udevadm trigger - let attempts-- - done +if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then + if [ "$ENABLE_CINDER" == "False" ]; then + # FIXME(yuanying): Use ephemeral disk for docker storage + # Currently Ironic doesn't support cinder volumes, + # so we must use preserved ephemeral disk instead of a cinder volume. + device_path=$(readlink -f /dev/disk/by-label/ephemeral0) + else + attempts=60 + while [ ${attempts} -gt 0 ]; do + device_name=$(ls /dev/disk/by-id | grep ${DOCKER_VOLUME:0:20}$) + if [ -n "${device_name}" ]; then + break + fi + echo "waiting for disk device" + sleep 0.5 + udevadm trigger + let attempts-- + done - if [ -z "${device_name}" ]; then - echo "ERROR: disk device does not exist" >&2 - exit 1 - fi + if [ -z "${device_name}" ]; then + echo "ERROR: disk device does not exist" >&2 + exit 1 + fi - device_path=/dev/disk/by-id/${device_name} + device_path=/dev/disk/by-id/${device_name} + fi fi $configure_docker_storage_driver diff --git a/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh b/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh index 9a9eb80e6..a28dc8f73 100644 --- a/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh +++ b/magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh @@ -15,9 +15,11 @@ configure_overlay () { rm -rf /var/lib/docker/* - mkfs.xfs -f ${device_path} - echo "${device_path} /var/lib/docker xfs defaults 0 0" >> /etc/fstab - mount -a + if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then + mkfs.xfs -f ${device_path} + echo "${device_path} /var/lib/docker xfs defaults 0 0" >> /etc/fstab + mount -a + fi echo "STORAGE_DRIVER=overlay" > /etc/sysconfig/docker-storage-setup @@ -31,8 +33,10 @@ configure_overlay () { configure_devicemapper () { clear_docker_storage_congiguration - pvcreate -f ${device_path} - vgcreate docker ${device_path} + if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then + pvcreate -f ${device_path} + vgcreate docker ${device_path} - echo "VG=docker" > /etc/sysconfig/docker-storage-setup + echo "VG=docker" > /etc/sysconfig/docker-storage-setup + fi } diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml index e0d555a19..4651ab3db 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml @@ -13,6 +13,7 @@ write_files: KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV" ENABLE_CINDER="$ENABLE_CINDER" DOCKER_VOLUME="$DOCKER_VOLUME" + DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" NETWORK_DRIVER="$NETWORK_DRIVER" FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml index 3801e9a19..d455a23e4 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml @@ -13,6 +13,7 @@ write_files: ETCD_SERVER_IP="$ETCD_SERVER_IP" ENABLE_CINDER="$ENABLE_CINDER" DOCKER_VOLUME="$DOCKER_VOLUME" + DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" NETWORK_DRIVER="$NETWORK_DRIVER" REGISTRY_ENABLED="$REGISTRY_ENABLED" diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index b303f8861..641af0f7f 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -92,7 +92,7 @@ parameters: description: > size of a cinder volume to allocate to docker for container/image storage - default: 25 + default: 0 docker_storage_driver: type: string diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 74131fb4c..66d779a0f 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -230,6 +230,7 @@ resources: "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} "$DOCKER_VOLUME": {get_resource: docker_volume} + "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$NETWORK_DRIVER": {get_param: network_driver} "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} @@ -442,12 +443,12 @@ resources: # docker_volume: - type: OS::Cinder::Volume + type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} docker_volume_attach: - type: OS::Cinder::VolumeAttachment + type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: kube_master} volume_id: {get_resource: docker_volume} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index 66e763197..a2f896344 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -227,6 +227,7 @@ resources: $KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} $ETCD_SERVER_IP: {get_param: etcd_server_ip} $DOCKER_VOLUME: {get_resource: docker_volume} + $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size} $DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver} $NETWORK_DRIVER: {get_param: network_driver} $REGISTRY_ENABLED: {get_param: registry_enabled} @@ -410,12 +411,12 @@ resources: # docker_volume: - type: OS::Cinder::Volume + type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} docker_volume_attach: - type: OS::Cinder::VolumeAttachment + type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: kube-minion} volume_id: {get_resource: docker_volume} diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml index 57278fbf8..baec083fc 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml @@ -100,7 +100,7 @@ parameters: description: > size of a cinder volume to allocate to docker for container/image storage - default: 25 + default: 0 docker_storage_driver: type: string @@ -430,6 +430,7 @@ resources: master_flavor: {get_param: master_flavor} external_network: {get_param: external_network} kube_allow_priv: {get_param: kube_allow_priv} + docker_volume_size: {get_param: docker_volume_size} docker_storage_driver: {get_param: docker_storage_driver} wait_condition_timeout: {get_param: wait_condition_timeout} network_driver: {get_param: network_driver} @@ -486,6 +487,7 @@ resources: etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]} external_network: {get_param: external_network} kube_allow_priv: {get_param: kube_allow_priv} + docker_volume_size: {get_param: docker_volume_size} docker_storage_driver: {get_param: docker_storage_driver} wait_condition_timeout: {get_param: wait_condition_timeout} registry_enabled: {get_param: registry_enabled} diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml index 67597f8e9..27f4fe2e5 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml @@ -35,6 +35,12 @@ parameters: constraints: - allowed_values: ["true", "false"] + docker_volume_size: + type: number + description: > + size of a cinder volume to allocate to docker for container/image + storage + docker_storage_driver: type: string description: docker storage driver name @@ -222,6 +228,7 @@ resources: "$KUBE_API_PORT": {get_param: kubernetes_port} "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} "$DOCKER_VOLUME": 'None' + "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$NETWORK_DRIVER": {get_param: network_driver} "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml index 1dc74598c..38d67667d 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml @@ -30,6 +30,12 @@ parameters: constraints: - allowed_values: ["true", "false"] + docker_volume_size: + type: number + description: > + size of a cinder volume to allocate to docker for container/image + storage + docker_storage_driver: type: string description: docker storage driver name @@ -219,6 +225,7 @@ resources: $KUBE_API_PORT: {get_param: kubernetes_port} $ETCD_SERVER_IP: {get_param: etcd_server_ip} $DOCKER_VOLUME: 'None' + $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size} $DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver} $NETWORK_DRIVER: {get_param: network_driver} $REGISTRY_ENABLED: {get_param: registry_enabled} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/template_def.py b/magnum/drivers/swarm_fedora_atomic_v1/template_def.py index 650f5b485..103c7db50 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/template_def.py +++ b/magnum/drivers/swarm_fedora_atomic_v1/template_def.py @@ -118,10 +118,19 @@ class AtomicSwarmTemplateDefinition(template_def.BaseTemplateDefinition): **kwargs) def get_env_files(self, cluster_template): + env_files = [] + + if cluster_template.docker_volume_size is None: + env_files.append('no_volume.yaml') + else: + env_files.append('with_volume.yaml') + if cluster_template.master_lb_enabled: - return [template_def.COMMON_ENV_PATH + 'with_master_lb.yaml'] + env_files.append('with_master_lb.yaml') else: - return [template_def.COMMON_ENV_PATH + 'no_master_lb.yaml'] + env_files.append('no_master_lb.yaml') + + return [template_def.COMMON_ENV_PATH + ef for ef in env_files] @property def driver_module_path(self): diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml index 0de667db3..4f893f614 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml @@ -118,7 +118,7 @@ parameters: description: > size of a cinder volume to allocate to docker for container/image storage - default: 25 + default: 0 docker_storage_driver: type: string diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.yaml index 1f8d5232b..0a3504cbe 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.yaml @@ -10,6 +10,7 @@ write_files: WAIT_CURL="$WAIT_CURL" ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL" DOCKER_VOLUME="$DOCKER_VOLUME" + DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" HTTP_PROXY="$HTTP_PROXY" HTTPS_PROXY="$HTTPS_PROXY" diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.yaml index e870b9644..d20165490 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.yaml @@ -9,6 +9,7 @@ write_files: WAIT_HANDLE_TOKEN="$WAIT_HANDLE_TOKEN" WAIT_CURL="$WAIT_CURL" DOCKER_VOLUME="$DOCKER_VOLUME" + DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE" DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" HTTP_PROXY="$HTTP_PROXY" HTTPS_PROXY="$HTTPS_PROXY" diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index 38e52f868..d67325391 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -204,6 +204,7 @@ resources: "$WAIT_HANDLE_TOKEN": {get_attr: [master_wait_handle, token]} "$WAIT_CURL": {get_attr: [master_wait_handle, curl_cli]} "$DOCKER_VOLUME": {get_resource: docker_volume} + "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$ETCD_DISCOVERY_URL": {get_param: discovery_url} "$HTTP_PROXY": {get_param: http_proxy} @@ -437,12 +438,12 @@ resources: # docker_volume: - type: OS::Cinder::Volume + type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} docker_volume_attach: - type: OS::Cinder::VolumeAttachment + type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: swarm_master} volume_id: {get_resource: docker_volume} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index 5ec65a401..f1dd4e2b8 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -189,6 +189,7 @@ resources: "$WAIT_HANDLE_TOKEN": {get_attr: [node_wait_handle, token]} "$WAIT_CURL": {get_attr: [node_wait_handle, curl_cli]} "$DOCKER_VOLUME": {get_resource: docker_volume} + "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} "$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver} "$HTTP_PROXY": {get_param: http_proxy} "$HTTPS_PROXY": {get_param: https_proxy} @@ -385,12 +386,12 @@ resources: # docker_volume: - type: OS::Cinder::Volume + type: Magnum::Optional::Cinder::Volume properties: size: {get_param: docker_volume_size} docker_volume_attach: - type: OS::Cinder::VolumeAttachment + type: Magnum::Optional::Cinder::VolumeAttachment properties: instance_uuid: {get_resource: swarm_node} volume_id: {get_resource: docker_volume} diff --git a/magnum/tests/unit/api/controllers/v1/test_baymodel.py b/magnum/tests/unit/api/controllers/v1/test_baymodel.py index cdbfaa389..58207824f 100644 --- a/magnum/tests/unit/api/controllers/v1/test_baymodel.py +++ b/magnum/tests/unit/api/controllers/v1/test_baymodel.py @@ -543,7 +543,6 @@ class TestPost(api_base.FunctionalTest): self._create_baymodel_raises_app_error(coe='osomatsu') def test_create_baymodel_with_invalid_docker_volume_size(self): - self._create_baymodel_raises_app_error(docker_volume_size=0) self._create_baymodel_raises_app_error(docker_volume_size=-1) self._create_baymodel_raises_app_error( docker_volume_size=1, diff --git a/magnum/tests/unit/api/controllers/v1/test_cluster_template.py b/magnum/tests/unit/api/controllers/v1/test_cluster_template.py index 277e64057..f22a52ca6 100644 --- a/magnum/tests/unit/api/controllers/v1/test_cluster_template.py +++ b/magnum/tests/unit/api/controllers/v1/test_cluster_template.py @@ -576,7 +576,6 @@ class TestPost(api_base.FunctionalTest): self._create_model_raises_app_error(coe='osomatsu') def test_create_cluster_template_with_invalid_docker_volume_size(self): - self._create_model_raises_app_error(docker_volume_size=0) self._create_model_raises_app_error(docker_volume_size=-1) self._create_model_raises_app_error( docker_volume_size=1, diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index 6da8e836e..570a55952 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -184,7 +184,8 @@ class TestClusterConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml', + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml'], env_files) @@ -256,7 +257,75 @@ class TestClusterConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml', + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml', + '../../common/templates/environments/disable_floating_ip.yaml'], + env_files) + + @patch('requests.get') + @patch('magnum.objects.ClusterTemplate.get_by_uuid') + @patch('magnum.drivers.common.driver.Driver.get_driver') + def test_extract_template_definition_only_required( + self, + mock_driver, + mock_objects_cluster_template_get_by_uuid, + mock_get): + + not_required = ['image_id', 'flavor_id', 'dns_nameserver', + 'docker_volume_size', 'fixed_network', 'http_proxy', + 'https_proxy', 'no_proxy', 'network_driver', + 'master_flavor_id', 'docker_storage_driver', + 'volume_driver'] + for key in not_required: + self.cluster_template_dict[key] = None + self.cluster_dict['discovery_url'] = 'https://discovery.etcd.io/test' + + cluster_template = objects.ClusterTemplate( + self.context, **self.cluster_template_dict) + mock_objects_cluster_template_get_by_uuid.return_value = \ + cluster_template + expected_result = str('{"action":"get","node":{"key":"test","value":' + '"1","modifiedIndex":10,"createdIndex":10}}') + mock_resp = mock.MagicMock() + mock_resp.text = expected_result + mock_get.return_value = mock_resp + mock_driver.return_value = k8s_dr.Driver() + cluster = objects.Cluster(self.context, **self.cluster_dict) + + (template_path, + definition, + env_files) = driver._extract_template_definition(self.context, + cluster) + + expected = { + 'auth_url': 'http://192.168.10.10:5000/v3', + 'cluster_uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', + 'discovery_url': 'https://discovery.etcd.io/test', + 'external_network': 'external_network_id', + 'flannel_backend': 'vxlan', + 'flannel_network_cidr': '10.101.0.0/16', + 'flannel_network_subnetlen': '26', + 'insecure_registry_url': '10.0.0.1:5000', + 'kube_version': 'fake-version', + 'magnum_url': 'http://127.0.0.1:9511/v1', + 'number_of_masters': 1, + 'number_of_minions': 1, + 'region_name': 'RegionOne', + 'registry_enabled': False, + 'ssh_key_name': 'keypair_id', + 'tenant_name': 'fake_tenant', + 'tls_disabled': False, + 'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de', + 'trustee_domain_id': 'trustee_domain_id', + 'trustee_password': 'fake_trustee_password', + 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', + 'trustee_username': 'fake_trustee', + 'username': 'fake_user' + } + self.assertEqual(expected, definition) + self.assertEqual( + ['../../common/templates/environments/no_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml'], env_files) @@ -411,17 +480,7 @@ class TestClusterConductorWithK8s(base.TestCase): @patch('requests.get') @patch('magnum.objects.ClusterTemplate.get_by_uuid') - def test_extract_template_definition_without_docker_volume_size( - self, - mock_objects_cluster_template_get_by_uuid, - mock_get): - self._test_extract_template_definition( - mock_objects_cluster_template_get_by_uuid, - mock_get, - missing_attr='docker_volume_size') - - @patch('requests.get') - @patch('magnum.objects.ClusterTemplate.get_by_uuid') + @patch('magnum.drivers.common.driver.Driver.get_driver') def test_extract_template_definition_without_docker_storage_driver( self, mock_objects_cluster_template_get_by_uuid, @@ -538,7 +597,8 @@ class TestClusterConductorWithK8s(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml', + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml', '../../common/templates/environments/disable_floating_ip.yaml'], env_files) reqget.assert_called_once_with('http://etcd/test?size=1') diff --git a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py index 9301309f1..48450cf9b 100644 --- a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py @@ -137,7 +137,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml'], env_files) @patch('requests.get') @@ -204,7 +205,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml'], env_files) @patch('requests.get') @@ -263,7 +265,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/no_volume.yaml', + '../../common/templates/environments/no_master_lb.yaml'], env_files) @patch('requests.get') @@ -324,7 +327,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/with_master_lb.yaml'], + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/with_master_lb.yaml'], env_files) @patch('requests.get') @@ -386,7 +390,8 @@ class TestClusterConductorWithSwarm(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/with_master_lb.yaml'], + ['../../common/templates/environments/with_volume.yaml', + '../../common/templates/environments/with_master_lb.yaml'], env_files) @patch('magnum.conductor.utils.retrieve_cluster_template') diff --git a/releasenotes/notes/no-cinder-volume-87b9339e066c30a0.yaml b/releasenotes/notes/no-cinder-volume-87b9339e066c30a0.yaml new file mode 100644 index 000000000..06300fdab --- /dev/null +++ b/releasenotes/notes/no-cinder-volume-87b9339e066c30a0.yaml @@ -0,0 +1,10 @@ +--- +prelude: > + Currently, the swarm and the kubernetes drivers use + a dedicated cinder volume to store the container + images. It was been observed that one cinder volume + per node is a bottleneck for large clusters. +fixes: + - Make the dedicated cinder volume per node an opt-in + option. By default, no cinder volumes will be created + unless the user passes the docker-volume-size argument. -- GitLab From ae2f9835629a4173dca6a1fdcbf5f857c0c51c16 Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Thu, 15 Dec 2016 16:36:00 +0100 Subject: [PATCH 4/9] [cern] install cern ca in swarm and k8s drop the cern ca certificates in the default location, so standard tools can do remote calls to CERN services without disable tls checks. Change-Id: I6ea9def9f1e75362c577d91995f5cf1a94c32e78 --- .../templates/fragments/install-cern-certs.sh | 22 +++++++++++++++++++ .../templates/kubemaster.yaml | 7 ++++++ .../templates/kubeminion.yaml | 7 ++++++ .../templates/swarmmaster.yaml | 7 ++++++ .../templates/swarmnode.yaml | 7 ++++++ 5 files changed, 50 insertions(+) create mode 100644 magnum/drivers/common/templates/fragments/install-cern-certs.sh diff --git a/magnum/drivers/common/templates/fragments/install-cern-certs.sh b/magnum/drivers/common/templates/fragments/install-cern-certs.sh new file mode 100644 index 000000000..5d4edabb5 --- /dev/null +++ b/magnum/drivers/common/templates/fragments/install-cern-certs.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +cd /etc/pki/ca-trust/source/anchors/ + +BASE_URL=https://cafiles.cern.ch/cafiles/certificates/ +CERTS=( "CERN Root Certification Authority 2" "CERN Grid Certification Authority" "CERN Certification Authority" ) + +for CERT in "${CERTS[@]}" +do + URL="$BASE_URL$CERT.crt" + curl "$( echo "$URL" | sed 's/ /%20/g' )" -o "$CERT.crt" + FIRST_LINE=$(head -1 "$CERT.crt") + if [[ ! $FIRST_LINE == *"BEGIN CERTIFICATE"* ]]; then + openssl x509 -inform DER -in "$CERT.crt" -out "$CERT.pem" + rm "$CERT.crt" + else + mv "$CERT.crt" "$CERT.pem" + fi +done + +update-ca-trust + diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 66d779a0f..2a74360aa 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -362,6 +362,12 @@ resources: group: ungrouped config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh} + install_cern_certs: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: ../../common/templates/fragments/install-cern-certs.sh} + kube_master_init: type: OS::Heat::MultipartMime properties: @@ -383,6 +389,7 @@ resources: - config: {get_resource: enable_kube_proxy} - config: {get_resource: kube_ui_service} - config: {get_resource: kube_examples} + - config: {get_resource: install_cern_certs} - config: {get_resource: master_wc_notify} ###################################################################### diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index a2f896344..5bf2c5c86 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -269,6 +269,12 @@ resources: group: ungrouped config: {get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh} + install_cern_certs: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: ../../common/templates/fragments/install-cern-certs.sh} + configure_docker_storage: type: OS::Heat::SoftwareConfig properties: @@ -362,6 +368,7 @@ resources: - config: {get_resource: enable_services} - config: {get_resource: enable_kube_proxy} - config: {get_resource: enable_docker_registry} + - config: {get_resource: install_cern_certs} - config: {get_resource: minion_wc_notify} ###################################################################### diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index d67325391..8b0985766 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -327,6 +327,12 @@ resources: params: "$NODE_SERVICES": "etcd docker.socket swarm-manager" + install_cern_certs: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: ../../common/templates/fragments/install-cern-certs.sh} + cfn_signal: type: "OS::Heat::SoftwareConfig" properties: @@ -370,6 +376,7 @@ resources: - config: {get_resource: write_swarm_master_service} - config: {get_resource: add_proxy} - config: {get_resource: enable_services} + - config: {get_resource: install_cern_certs} - config: {get_resource: cfn_signal} - config: {get_resource: volume_service} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index f1dd4e2b8..3923cc699 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -322,6 +322,12 @@ resources: group: ungrouped config: {get_file: fragments/volume-service.sh} + install_cern_certs: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: ../../common/templates/fragments/install-cern-certs.sh} + swarm_node_init: type: "OS::Heat::MultipartMime" properties: @@ -340,6 +346,7 @@ resources: - config: {get_resource: add_proxy} - config: {get_resource: enable_docker_registry} - config: {get_resource: enable_services} + - config: {get_resource: install_cern_certs} - config: {get_resource: cfn_signal} - config: {get_resource: volume_service} -- GitLab From 52cf5387c958e4a6d209b479623b90db31fbd284 Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Thu, 15 Dec 2016 16:42:40 +0100 Subject: [PATCH 5/9] [cern] add skydns for kubernetes cluster dns add by default an additional pod with skydns, and configure kubernetes to rely on it for the cluster internal dns resolve. Change-Id: I9af4a1ccc5e068090cf4cdcd7be7105015130bc9 --- .../fragments/configure-kubernetes-minion.sh | 4 +- .../kubernetes/fragments/kube-dns-service.sh | 225 ++++++++++++++++++ .../templates/kubemaster.yaml | 7 + 3 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 magnum/drivers/common/templates/kubernetes/fragments/kube-dns-service.sh diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 997b182d7..eea4de84c 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -31,8 +31,8 @@ sed -i ' # The hostname of the node is set to be the Nova name of the instance, and # the option --hostname-override for kubelet uses the hostname to register the node. # Using any other name will break the load balancer and cinder volume features. -HOSTNAME=$(hostname --short | sed 's/\.novalocal//') -KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 ${KUBE_CONFIG} --hostname-override=${HOSTNAME}" +HOSTNAME=$(hostname -I | cut -d' ' -f1) +KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 --hostname-override=${HOSTNAME} --cluster-dns=10.254.10.10 --cluster-domain=cluster.local ${KUBE_CONFIG}" if [ -n "${INSECURE_REGISTRY_URL}" ]; then KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-dns-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-dns-service.sh new file mode 100644 index 000000000..2f4eeb8c8 --- /dev/null +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-dns-service.sh @@ -0,0 +1,225 @@ +#!/bin/sh + +# this service is required because docker will start only after cloud init was finished +# due to the service dependencies in Fedora Atomic (docker <- docker-storage-setup <- cloud-final) + + +. /etc/sysconfig/heat-params + +KUBE_DNS_RC=/srv/kubernetes/manifests/kube-skydns-rc.yaml +[ -f ${KUBE_DNS_RC} ] || { + echo "Writing File: $KUBE_DNS_RC" + mkdir -p $(dirname ${KUBE_DNS_RC}) + cat << EOF > ${KUBE_DNS_RC} +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-dns-v11 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v11 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-dns + version: v11 + template: + metadata: + labels: + k8s-app: kube-dns + version: v11 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: etcd + image: gcr.io/google_containers/etcd-amd64:2.2.1 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + memory: 500Mi + requests: + cpu: 100m + memory: 50Mi + command: + - /usr/local/bin/etcd + - -data-dir + - /var/etcd/data + - -listen-client-urls + - http://127.0.0.1:2379,http://127.0.0.1:4001 + - -advertise-client-urls + - http://127.0.0.1:2379,http://127.0.0.1:4001 + - -initial-cluster-token + - skydns-etcd + volumeMounts: + - name: etcd-storage + mountPath: /var/etcd/data + - name: kube2sky + image: gcr.io/google_containers/kube2sky:1.14 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + # Kube2sky watches all pods. + memory: 200Mi + requests: + cpu: 100m + memory: 50Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 30 + timeoutSeconds: 5 + args: + # command = "/kube2sky" + - --domain=cluster.local + - --kubecfg-file=/srv/kubernetes/kubeconfig.yaml + volumeMounts: + - mountPath: /srv/kubernetes/ + name: config-vol + - name: skydns + image: gcr.io/google_containers/skydns:2015-10-13-8c72f8c + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 100m + memory: 50Mi + args: + # command = "/skydns" + - -machines=http://127.0.0.1:4001 + - -addr=0.0.0.0:53 + - -ns-rotate=false + - -domain=cluster.local. + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: gcr.io/google_containers/exechealthz:1.0 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + args: + - -cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null + - -port=8080 + ports: + - containerPort: 8080 + protocol: TCP + volumes: + - name: etcd-storage + emptyDir: {} + - name: config-vol + hostPath: + path: /srv/kubernetes/ + dnsPolicy: Default # Don't use cluster DNS. +EOF +} + +KUBE_DNS_SVC=/srv/kubernetes/manifests/kube-skydns-svc.yaml +[ -f ${KUBE_DNS_SVC} ] || { + echo "Writing File: $KUBE_DNS_SVC" + mkdir -p $(dirname ${KUBE_DNS_SVC}) + cat << EOF > ${KUBE_DNS_SVC} +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.254.10.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP +EOF +} + +KUBE_DNS_BIN=/usr/local/bin/kube-dns +[ -f ${KUBE_DNS_BIN} ] || { + echo "Writing File: $KUBE_DNS_BIN" + mkdir -p $(dirname ${KUBE_DNS_BIN}) + cat << EOF > ${KUBE_DNS_BIN} +#!/bin/sh +until curl -sf "http://127.0.0.1:8080/healthz" +do + echo "Waiting for Kubernetes API..." + sleep 5 +done + +/usr/bin/kubectl create -f $KUBE_DNS_RC --namespace=kube-system +/usr/bin/kubectl create -f $KUBE_DNS_SVC --namespace=kube-system +EOF +} + +KUBE_DNS_SERVICE=/etc/systemd/system/kube-dns.service +[ -f ${KUBE_DNS_SERVICE} ] || { + echo "Writing File: $KUBE_DNS_SERVICE" + mkdir -p $(dirname ${KUBE_DNS_SERVICE}) + cat << EOF > ${KUBE_DNS_SERVICE} +[Unit] +After=kube-apiserver.service +Requires=kube-apiserver.service + +[Service] +Type=oneshot +Environment=HOME=/root +EnvironmentFile=-/etc/kubernetes/config +ExecStart=${KUBE_DNS_BIN} + +[Install] +WantedBy=multi-user.target +EOF +} + +chown root:root ${KUBE_DNS_BIN} +chmod 0755 ${KUBE_DNS_BIN} + +chown root:root ${KUBE_DNS_SERVICE} +chmod 0644 ${KUBE_DNS_SERVICE} + +systemctl enable kube-dns +systemctl start --no-block kube-dns diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 2a74360aa..d916bd35b 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -332,6 +332,12 @@ resources: group: ungrouped config: {get_file: ../../common/templates/kubernetes/fragments/kube-system-namespace-service.sh} + kube_dns_service: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: ../../common/templates/kubernetes/fragments/kube-dns-service.sh} + kube_ui_service: type: OS::Heat::SoftwareConfig properties: @@ -385,6 +391,7 @@ resources: - config: {get_resource: network_config_service} - config: {get_resource: network_service} - config: {get_resource: kube_system_namespace_service} + - config: {get_resource: kube_dns_service} - config: {get_resource: enable_kube_podmaster} - config: {get_resource: enable_kube_proxy} - config: {get_resource: kube_ui_service} -- GitLab From b1519cafee16cae2dfcc3c4aa78330a1275623b5 Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Thu, 15 Dec 2016 16:46:26 +0100 Subject: [PATCH 6/9] [cern] disable cern-services (no dns wait on boot) set metadata property cern-services to false to all master and slave nodes in swarm, kubernetes and mesos. this prevents nova from waiting until the node appears in the cern dns before considering it active. we don't rely on dns for any part of the magnum installation. Change-Id: If9898b6386c8f753eb51d9fb04932d2238bd4791 --- magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml | 2 ++ magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml | 2 ++ magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml | 2 ++ magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml | 2 ++ .../drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml | 2 ++ magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml | 2 ++ 6 files changed, 12 insertions(+) diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index d916bd35b..4ebc4f8ab 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -412,6 +412,8 @@ resources: key_name: {get_param: ssh_key_name} user_data_format: RAW user_data: {get_resource: kube_master_init} + metadata: + cern-services: false networks: - port: {get_resource: kube_master_eth0} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index 5bf2c5c86..0a850f097 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -390,6 +390,8 @@ resources: key_name: {get_param: ssh_key_name} user_data_format: RAW user_data: {get_resource: kube_minion_init} + metadata: + cern-services: false networks: - port: {get_resource: kube_minion_eth0} diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml index 1ae7ad9a5..492200b0a 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml @@ -57,6 +57,8 @@ resources: flavor: {get_param: master_flavor} key_name: {get_param: ssh_key_name} user_data_format: SOFTWARE_CONFIG + metadata: + cern-services: false networks: - port: {get_resource: mesos_master_eth0} diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml index d03da4e1e..1a2a51adc 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml @@ -243,6 +243,8 @@ resources: key_name: {get_param: ssh_key_name} user_data_format: RAW user_data: {get_resource: mesos_slave_init} + metadata: + cern-services: false networks: - port: {get_resource: mesos_slave_eth0} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index 8b0985766..e65e46c90 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -397,6 +397,8 @@ resources: get_param: ssh_key_name user_data_format: RAW user_data: {get_resource: swarm_master_init} + metadata: + cern-services: false networks: - port: get_resource: swarm_master_eth0 diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index 3923cc699..bfb6be5d1 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -361,6 +361,8 @@ resources: get_param: ssh_key_name user_data_format: RAW user_data: {get_resource: swarm_node_init} + metadata: + cern-services: false networks: - port: get_resource: swarm_node_eth0 -- GitLab From 94edcf5aa8b24609054bfc60164aa38452ef3788 Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Thu, 15 Dec 2016 16:56:41 +0100 Subject: [PATCH 7/9] [cern] replace _ with - in all node names nova integrates with cern dns, which does not allows '_' in device names. rename all magnum master and slave nodes on all drivers to use '-' instead. Change-Id: Ib2a05469e409f053e65a3d62b714fc5e263aae4c --- .../drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml | 6 +++--- magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml | 2 +- magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml | 4 ++-- .../swarm_fedora_atomic_v1/templates/swarmmaster.yaml | 6 +++--- .../drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 4ebc4f8ab..1994be9aa 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -190,7 +190,7 @@ resources: master_wait_condition: type: OS::Heat::WaitCondition - depends_on: kube_master + depends_on: kube-master properties: handle: {get_resource: master_wait_handle} timeout: {get_param: wait_condition_timeout} @@ -404,7 +404,7 @@ resources: # a single kubernetes master. # - kube_master: + kube-master: type: OS::Nova::Server properties: image: {get_param: server_image} @@ -466,7 +466,7 @@ resources: docker_volume_attach: type: Magnum::Optional::Cinder::VolumeAttachment properties: - instance_uuid: {get_resource: kube_master} + instance_uuid: {get_resource: kube-master} volume_id: {get_resource: docker_volume} mountpoint: /dev/vdb diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml index 492200b0a..eee70e0ec 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml @@ -50,7 +50,7 @@ resources: # Mesos master server. # - mesos_master: + mesos-master: type: OS::Nova::Server properties: image: {get_param: server_image} diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml index 1a2a51adc..3e48ea60c 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml @@ -138,7 +138,7 @@ resources: slave_wait_condition: type: OS::Heat::WaitCondition - depends_on: mesos_slave + depends_on: mesos-slave properties: handle: {get_resource: slave_wait_handle} timeout: {get_param: wait_condition_timeout} @@ -235,7 +235,7 @@ resources: # a single Mesos slave. # - mesos_slave: + mesos-slave: type: OS::Nova::Server properties: image: {get_param: server_image} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index e65e46c90..bafc7d32f 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -162,7 +162,7 @@ resources: master_wait_condition: type: "OS::Heat::WaitCondition" - depends_on: swarm_master + depends_on: swarm-master properties: handle: {get_resource: master_wait_handle} timeout: 6000 @@ -386,7 +386,7 @@ resources: # side the swarm agent. # - swarm_master: + swarm-master: type: "OS::Nova::Server" properties: image: @@ -454,7 +454,7 @@ resources: docker_volume_attach: type: Magnum::Optional::Cinder::VolumeAttachment properties: - instance_uuid: {get_resource: swarm_master} + instance_uuid: {get_resource: swarm-master} volume_id: {get_resource: docker_volume} mountpoint: /dev/vdb diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index bfb6be5d1..4bec5793d 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -168,7 +168,7 @@ resources: node_wait_condition: type: "OS::Heat::WaitCondition" - depends_on: swarm_node + depends_on: swarm-node properties: handle: {get_resource: node_wait_handle} timeout: 6000 @@ -350,7 +350,7 @@ resources: - config: {get_resource: cfn_signal} - config: {get_resource: volume_service} - swarm_node: + swarm-node: type: "OS::Nova::Server" properties: image: @@ -402,7 +402,7 @@ resources: docker_volume_attach: type: Magnum::Optional::Cinder::VolumeAttachment properties: - instance_uuid: {get_resource: swarm_node} + instance_uuid: {get_resource: swarm-node} volume_id: {get_resource: docker_volume} mountpoint: /dev/vdb -- GitLab From b077fbf69f55d40ae880e183a617c14958b21609 Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Thu, 15 Dec 2016 17:08:17 +0100 Subject: [PATCH 8/9] [cern] drop dependency on neutron objects drop dependency on neutron networks, subnets, routers, floating ips, security groups, load balancers. we don't currently have neutron enabled everywhere, and for several cases these concepts are not yet supported by our neutron setup (routers, floating ips, security groups, load balancers) or are not available to users (networks, subnets). for cases where the node ip is required, rely instead on the first_ip exposed by the heat resource. done for kubernetes, swarm and mesos. Change-Id: I9c37053b6b1f381873766b72b260bd6eba622429 --- .../fragments/write-heat-params-master.sh | 42 ++++ .../fragments/write-kubeconfig.yaml | 1 + .../templates/kubecluster.yaml | 211 ++---------------- .../templates/kubemaster.yaml | 65 +----- .../templates/kubeminion.yaml | 37 +-- .../fragments/configure-mesos-master.sh | 5 + .../templates/mesoscluster.yaml | 130 +---------- .../templates/mesosmaster.yaml | 33 +-- .../mesos_ubuntu_v1/templates/mesosslave.yaml | 31 +-- .../templates/cluster.yaml | 190 ++-------------- .../templates/fragments/make-cert.py | 8 +- .../fragments/write-heat-params-master.sh | 39 ++++ .../fragments/write-heat-params-node.sh | 39 ++++ .../fragments/write-swarm-master-service.sh | 4 +- .../templates/swarmmaster.yaml | 82 +------ .../templates/swarmnode.yaml | 32 +-- 16 files changed, 208 insertions(+), 741 deletions(-) create mode 100644 magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh create mode 100644 magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.sh create mode 100644 magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.sh diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh new file mode 100644 index 000000000..a787cbe0f --- /dev/null +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh @@ -0,0 +1,42 @@ +#!/bin/sh +KUBE_API_PUBLIC_ADDRESS_TMP=$KUBE_API_PUBLIC_ADDRESS +KUBE_API_PRIVATE_ADDRESS_TMP=$KUBE_API_PRIVATE_ADDRESS +KUBE_NODE_IP_TMP=$KUBE_NODE_IP + +cat > /etc/sysconfig/heat-params << END +KUBE_API_PUBLIC_ADDRESS="${KUBE_API_PUBLIC_ADDRESS_TMP:-$(hostname -I | cut -d' ' -f1)}" +KUBE_API_PRIVATE_ADDRESS="${KUBE_API_PRIVATE_ADDRESS_TMP:-$(hostname -I | cut -d' ' -f1)}" +KUBE_API_PORT="$KUBE_API_PORT" +KUBE_NODE_PUBLIC_IP="${KUBE_NODE_IP_TMP:-$(hostname -I | cut -d' ' -f1)}" +KUBE_NODE_IP="${KUBE_NODE_IP_TMP:-$(hostname -I | cut -d' ' -f1)}" +KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV" +ENABLE_CINDER="$ENABLE_CINDER" +DOCKER_VOLUME="$DOCKER_VOLUME" +DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" +NETWORK_DRIVER="$NETWORK_DRIVER" +FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR" +FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN" +FLANNEL_BACKEND="$FLANNEL_BACKEND" +PORTAL_NETWORK_CIDR="$PORTAL_NETWORK_CIDR" +ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL" +USERNAME="$USERNAME" +PASSWORD="$PASSWORD" +TENANT_NAME="$TENANT_NAME" +CLUSTER_SUBNET="$CLUSTER_SUBNET" +TLS_DISABLED="$TLS_DISABLED" +CLUSTER_UUID="$CLUSTER_UUID" +MAGNUM_URL="$MAGNUM_URL" +HTTP_PROXY="$HTTP_PROXY" +HTTPS_PROXY="$HTTPS_PROXY" +NO_PROXY="$NO_PROXY" +WAIT_CURL="$WAIT_CURL" +KUBE_VERSION="$KUBE_VERSION" +TRUSTEE_USER_ID="$TRUSTEE_USER_ID" +TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD" +TRUST_ID="$TRUST_ID" +AUTH_URL="$AUTH_URL" +INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL" +END + +chown root:root /etc/sysconfig/heat-params +chmod 644 /etc/sysconfig/heat-params diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-kubeconfig.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-kubeconfig.yaml index 659adfbfa..45bf2cfbc 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-kubeconfig.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-kubeconfig.yaml @@ -15,6 +15,7 @@ write_files: clusters: - name: kubernetes cluster: + server: https://$KUBE_MASTER_IP:$KUBE_API_PORT certificate-authority: CA_CERT contexts: - context: diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 641af0f7f..3befb49c6 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -265,185 +265,6 @@ parameters: resources: - ###################################################################### - # - # network resources. allocate a network and router for our server. - # Important: the Load Balancer feature in Kubernetes requires that - # the name for the fixed_network must be "private" for the - # address lookup in Kubernetes to work properly - # - - fixed_network: - type: OS::Neutron::Net - properties: - name: private - - fixed_subnet: - type: OS::Neutron::Subnet - properties: - cidr: {get_param: fixed_network_cidr} - network: {get_resource: fixed_network} - dns_nameservers: - - {get_param: dns_nameserver} - - extrouter: - type: OS::Neutron::Router - properties: - external_gateway_info: - network: {get_param: external_network} - - extrouter_inside: - type: OS::Neutron::RouterInterface - properties: - router_id: {get_resource: extrouter} - subnet: {get_resource: fixed_subnet} - - ###################################################################### - # - # security groups. we need to permit network traffic of various - # sorts. - # - - secgroup_base: - type: OS::Neutron::SecurityGroup - properties: - rules: - - protocol: icmp - - protocol: tcp - port_range_min: 22 - port_range_max: 22 - - secgroup_kube_master: - type: OS::Neutron::SecurityGroup - properties: - rules: - - protocol: tcp - port_range_min: 7080 - port_range_max: 7080 - - protocol: tcp - port_range_min: 8080 - port_range_max: 8080 - - protocol: tcp - port_range_min: 2379 - port_range_max: 2379 - - protocol: tcp - port_range_min: 2380 - port_range_max: 2380 - - protocol: tcp - port_range_min: 6443 - port_range_max: 6443 - - protocol: tcp - port_range_min: 30000 - port_range_max: 32767 - - secgroup_kube_minion: - type: OS::Neutron::SecurityGroup - properties: - rules: - - protocol: icmp - - protocol: tcp - - protocol: udp - - ###################################################################### - # - # load balancers. - # - api_loadbalancer: - type: Magnum::Optional::Neutron::LBaaS::LoadBalancer - properties: - vip_subnet: {get_resource: fixed_subnet} - - api_listener: - type: Magnum::Optional::Neutron::LBaaS::Listener - properties: - loadbalancer: {get_resource: api_loadbalancer} - protocol: {get_param: loadbalancing_protocol} - protocol_port: {get_param: kubernetes_port} - - api_pool: - type: Magnum::Optional::Neutron::LBaaS::Pool - properties: - lb_algorithm: ROUND_ROBIN - listener: {get_resource: api_listener} - protocol: {get_param: loadbalancing_protocol} - - api_monitor: - type: Magnum::Optional::Neutron::LBaaS::HealthMonitor - properties: - type: TCP - delay: 5 - max_retries: 5 - timeout: 5 - pool: { get_resource: api_pool } - - api_pool_floating: - type: Magnum::Optional::Neutron::FloatingIP - depends_on: - - extrouter_inside - properties: - floating_network: {get_param: external_network} - port_id: {get_attr: [api_loadbalancer, vip_port_id]} - - etcd_loadbalancer: - type: Magnum::Optional::Neutron::LBaaS::LoadBalancer - properties: - vip_subnet: {get_resource: fixed_subnet} - - etcd_listener: - type: Magnum::Optional::Neutron::LBaaS::Listener - properties: - loadbalancer: {get_resource: etcd_loadbalancer} - protocol: HTTP - protocol_port: 2379 - - etcd_pool: - type: Magnum::Optional::Neutron::LBaaS::Pool - properties: - lb_algorithm: ROUND_ROBIN - listener: {get_resource: etcd_listener} - protocol: HTTP - - etcd_monitor: - type: Magnum::Optional::Neutron::LBaaS::HealthMonitor - properties: - type: TCP - delay: 5 - max_retries: 5 - timeout: 5 - pool: { get_resource: etcd_pool } - - ###################################################################### - # - # resources that expose the IPs of either the kube master or a given - # LBaaS pool depending on whether LBaaS is enabled for the cluster. - # - - api_address_lb_switch: - type: Magnum::ApiGatewaySwitcher - properties: - pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]} - pool_private_ip: {get_attr: [api_loadbalancer, vip_address]} - master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]} - master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]} - - etcd_address_lb_switch: - type: Magnum::ApiGatewaySwitcher - properties: - pool_private_ip: {get_attr: [etcd_loadbalancer, vip_address]} - master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]} - - ###################################################################### - # - # resources that expose the IPs of either floating ip or a given - # fixed ip depending on whether FloatingIP is enabled for the cluster. - # - - api_address_floating_switch: - type: Magnum::FloatingIPAddressSwitcher - properties: - public_ip: {get_attr: [api_address_lb_switch, public_ip]} - private_ip: {get_attr: [api_address_lb_switch, private_ip]} - ###################################################################### # # kubernetes masters. This is a resource group that will create @@ -452,15 +273,13 @@ resources: kube_masters: type: OS::Heat::ResourceGroup - depends_on: - - extrouter_inside properties: count: {get_param: number_of_masters} resource_def: type: kubemaster.yaml properties: - api_public_address: {get_attr: [api_pool_floating, floating_ip_address]} - api_private_address: {get_attr: [api_loadbalancer, vip_address]} + api_public_address: "" + api_private_address: "" ssh_key_name: {get_param: ssh_key_name} server_image: {get_param: server_image} master_flavor: {get_param: master_flavor} @@ -477,17 +296,17 @@ resources: discovery_url: {get_param: discovery_url} cluster_uuid: {get_param: cluster_uuid} magnum_url: {get_param: magnum_url} - fixed_network: {get_resource: fixed_network} - fixed_subnet: {get_resource: fixed_subnet} - api_pool_id: {get_resource: api_pool} - etcd_pool_id: {get_resource: etcd_pool} + fixed_network: "" + fixed_subnet: "" + api_pool_id: "" + etcd_pool_id: "" username: {get_param: username} password: {get_param: password} tenant_name: {get_param: tenant_name} kubernetes_port: {get_param: kubernetes_port} tls_disabled: {get_param: tls_disabled} - secgroup_base_id: {get_resource: secgroup_base} - secgroup_kube_master_id: {get_resource: secgroup_kube_master} + secgroup_base_id: "" + secgroup_kube_master_id: "" http_proxy: {get_param: http_proxy} https_proxy: {get_param: https_proxy} no_proxy: {get_param: no_proxy} @@ -506,8 +325,6 @@ resources: kube_minions: type: OS::Heat::ResourceGroup - depends_on: - - extrouter_inside properties: count: {get_param: number_of_minions} removal_policies: [{resource_list: {get_param: minions_to_remove}}] @@ -517,12 +334,12 @@ resources: ssh_key_name: {get_param: ssh_key_name} server_image: {get_param: server_image} minion_flavor: {get_param: minion_flavor} - fixed_network: {get_resource: fixed_network} - fixed_subnet: {get_resource: fixed_subnet} + fixed_network: "" + fixed_subnet: "" network_driver: {get_param: network_driver} flannel_network_cidr: {get_param: flannel_network_cidr} - kube_master_ip: {get_attr: [api_address_lb_switch, private_ip]} - etcd_server_ip: {get_attr: [etcd_address_lb_switch, private_ip]} + kube_master_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]} + etcd_server_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]} external_network: {get_param: external_network} kube_allow_priv: {get_param: kube_allow_priv} docker_volume_size: {get_param: docker_volume_size} @@ -544,7 +361,7 @@ resources: password: {get_param: password} kubernetes_port: {get_param: kubernetes_port} tls_disabled: {get_param: tls_disabled} - secgroup_kube_minion_id: {get_resource: secgroup_kube_minion} + secgroup_kube_minion_id: "" http_proxy: {get_param: http_proxy} https_proxy: {get_param: https_proxy} no_proxy: {get_param: no_proxy} @@ -564,7 +381,7 @@ outputs: str_replace: template: api_ip_address params: - api_ip_address: {get_attr: [api_address_floating_switch, ip_address]} + api_ip_address: {get_attr: [kube_masters, resource.0.kube_master_external_ip]} description: > This is the API endpoint of the Kubernetes cluster. Use this to access the Kubernetes API. diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 1994be9aa..8b4cf6eba 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -195,20 +195,6 @@ resources: handle: {get_resource: master_wait_handle} timeout: {get_param: wait_condition_timeout} - ###################################################################### - # - # resource that exposes the IPs of either the kube master or the API - # LBaaS pool depending on whether LBaaS is enabled for the cluster. - # - - api_address_switch: - type: Magnum::ApiGatewaySwitcher - properties: - pool_public_ip: {get_param: api_public_address} - pool_private_ip: {get_param: api_private_address} - master_public_ip: {get_attr: [kube_master_floating, floating_ip_address]} - master_private_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} - ###################################################################### # # software configs. these are components that are combined into @@ -221,13 +207,13 @@ resources: group: ungrouped config: str_replace: - template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.yaml} + template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.sh} params: - "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]} - "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]} + "$KUBE_API_PUBLIC_ADDRESS": "" + "$KUBE_API_PRIVATE_ADDRESS": "" "$KUBE_API_PORT": {get_param: kubernetes_port} - "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]} - "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + "$KUBE_NODE_PUBLIC_IP": "" + "$KUBE_NODE_IP": "" "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} "$DOCKER_VOLUME": {get_resource: docker_volume} "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size} @@ -414,43 +400,6 @@ resources: user_data: {get_resource: kube_master_init} metadata: cern-services: false - networks: - - port: {get_resource: kube_master_eth0} - - kube_master_eth0: - type: OS::Neutron::Port - properties: - network: {get_param: fixed_network} - security_groups: - - {get_param: secgroup_base_id} - - {get_param: secgroup_kube_master_id} - fixed_ips: - - subnet: {get_param: fixed_subnet} - allowed_address_pairs: - - ip_address: {get_param: flannel_network_cidr} - replacement_policy: AUTO - - kube_master_floating: - type: Magnum::Optional::KubeMaster::Neutron::FloatingIP - properties: - floating_network: {get_param: external_network} - port_id: {get_resource: kube_master_eth0} - - api_pool_member: - type: Magnum::Optional::Neutron::LBaaS::PoolMember - properties: - pool: {get_param: api_pool_id} - address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} - subnet: { get_param: fixed_subnet } - protocol_port: {get_param: kubernetes_port} - - etcd_pool_member: - type: Magnum::Optional::Neutron::LBaaS::PoolMember - properties: - pool: {get_param: etcd_pool_id} - address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} - subnet: { get_param: fixed_subnet } - protocol_port: 2379 ###################################################################### # @@ -473,11 +422,11 @@ resources: outputs: kube_master_ip: - value: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + value: {get_attr: [kube-master, first_address]} description: > This is the "private" IP address of the Kubernetes master node. kube_master_external_ip: - value: {get_attr: [kube_master_floating, floating_ip_address]} + value: {get_attr: [kube-master, first_address]} description: > This is the "public" IP address of the Kubernetes master node. diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index 0a850f097..4811e833a 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -223,8 +223,8 @@ resources: $KUBE_ALLOW_PRIV: {get_param: kube_allow_priv} $KUBE_MASTER_IP: {get_param: kube_master_ip} $KUBE_API_PORT: {get_param: kubernetes_port} - $KUBE_NODE_PUBLIC_IP: {get_attr: [kube_minion_floating, floating_ip_address]} - $KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} + $KUBE_NODE_PUBLIC_IP: "" + $KUBE_NODE_IP: "" $ETCD_SERVER_IP: {get_param: etcd_server_ip} $DOCKER_VOLUME: {get_resource: docker_volume} $DOCKER_VOLUME_SIZE: {get_param: docker_volume_size} @@ -261,7 +261,12 @@ resources: type: OS::Heat::SoftwareConfig properties: group: ungrouped - config: {get_file: ../../common/templates/kubernetes/fragments/write-kubeconfig.yaml} + config: + str_replace: + template: {get_file: ../../common/templates/kubernetes/fragments/write-kubeconfig.yaml} + params: + $KUBE_MASTER_IP: {get_param: kube_master_ip} + $KUBE_API_PORT: {get_param: kubernetes_port} make_cert: type: OS::Heat::SoftwareConfig @@ -392,26 +397,6 @@ resources: user_data: {get_resource: kube_minion_init} metadata: cern-services: false - networks: - - port: {get_resource: kube_minion_eth0} - - kube_minion_eth0: - type: OS::Neutron::Port - properties: - network: {get_param: fixed_network} - security_groups: - - get_param: secgroup_kube_minion_id - fixed_ips: - - subnet: {get_param: fixed_subnet} - allowed_address_pairs: - - ip_address: {get_param: flannel_network_cidr} - replacement_policy: AUTO - - kube_minion_floating: - type: Magnum::Optional::KubeMinion::Neutron::FloatingIP - properties: - floating_network: {get_param: external_network} - port_id: {get_resource: kube_minion_eth0} ###################################################################### # @@ -434,16 +419,16 @@ resources: outputs: kube_minion_ip: - value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} + value: {get_attr: [kube-minion, first_address]} description: > This is the "public" IP address of the Kubernetes minion node. kube_minion_external_ip: - value: {get_attr: [kube_minion_floating, floating_ip_address]} + value: {get_attr: [kube-minion, first_address]} description: > This is the "public" IP address of the Kubernetes minion node. OS::stack_id: - value: {get_param: "OS::stack_id"} + value: {get_attr: [kube-minion, first_address]} description: > This is a id of the stack which creates from this template. diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/fragments/configure-mesos-master.sh b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/configure-mesos-master.sh index fd7b66ca9..f7549d02d 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/fragments/configure-mesos-master.sh +++ b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/configure-mesos-master.sh @@ -10,6 +10,11 @@ myip=$(ip addr show eth0 | # Fix /etc/hosts sed -i "s/127.0.1.1/$myip/" /etc/hosts + # 2016/08/18 benoel + # When started with cern-services false, Marathon does not advertise itself + # on the correct IP. It uses 127.0.0.1 + echo "LIBPROCESS_IP=$myip" > /etc/default/marathon + ###################################################################### # # Configure ZooKeeper diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml index f5506b132..65dc3327b 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml @@ -199,62 +199,6 @@ parameters: resources: - ###################################################################### - # - # network resources. allocate a network and router for our server. - # - - fixed_network: - type: OS::Neutron::Net - - fixed_subnet: - type: OS::Neutron::Subnet - properties: - cidr: {get_param: fixed_network_cidr} - network: {get_resource: fixed_network} - dns_nameservers: - - {get_param: dns_nameserver} - - extrouter: - type: OS::Neutron::Router - properties: - external_gateway_info: - network: {get_param: external_network} - - extrouter_inside: - type: OS::Neutron::RouterInterface - properties: - router_id: {get_resource: extrouter} - subnet: {get_resource: fixed_subnet} - - ###################################################################### - # - # security groups. we need to permit network traffic of various - # sorts. - # - - secgroup_base: - type: OS::Neutron::SecurityGroup - properties: - rules: - - protocol: icmp - - protocol: tcp - port_range_min: 22 - port_range_max: 22 - - protocol: tcp - remote_mode: remote_group_id - - secgroup_mesos: - type: OS::Neutron::SecurityGroup - properties: - rules: - - protocol: tcp - port_range_min: 5050 - port_range_max: 5050 - - protocol: tcp - port_range_min: 8080 - port_range_max: 8080 - ###################################################################### # # Master SoftwareConfig. @@ -339,58 +283,6 @@ resources: config: {get_resource: start_services_master} servers: {get_attr: [mesos_masters, attributes, mesos_server_id]} - ###################################################################### - # - # load balancers. - # - api_loadbalancer: - type: Magnum::Optional::Neutron::LBaaS::LoadBalancer - properties: - vip_subnet: {get_resource: fixed_subnet} - - api_listener: - type: Magnum::Optional::Neutron::LBaaS::Listener - properties: - loadbalancer: {get_resource: api_loadbalancer} - protocol: HTTP - protocol_port: 8080 - - api_pool: - type: Magnum::Optional::Neutron::LBaaS::Pool - properties: - lb_algorithm: ROUND_ROBIN - listener: {get_resource: api_listener} - protocol: HTTP - - api_monitor: - type: Magnum::Optional::Neutron::LBaaS::HealthMonitor - properties: - type: TCP - delay: 5 - max_retries: 5 - timeout: 5 - pool: { get_resource: api_pool } - - api_pool_floating: - type: Magnum::Optional::Neutron::FloatingIP - depends_on: - - extrouter_inside - properties: - floating_network: {get_param: external_network} - port_id: {get_attr: [api_loadbalancer, vip_port_id]} - - ###################################################################### - # - # resources that expose the IPs of either the mesos master or a given - # LBaaS pool depending on whether LBaaS is enabled for the bay. - # - - api_address_switch: - type: Magnum::ApiGatewaySwitcher - properties: - pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]} - master_public_ip: {get_attr: [mesos_masters, resource.0.mesos_master_external_ip]} - ###################################################################### # # Mesos masters. This is a resource group that will create @@ -399,8 +291,6 @@ resources: mesos_masters: type: OS::Heat::ResourceGroup - depends_on: - - extrouter_inside properties: count: {get_param: number_of_masters} resource_def: @@ -410,11 +300,11 @@ resources: server_image: {get_param: server_image} master_flavor: {get_param: master_flavor} external_network: {get_param: external_network} - fixed_network: {get_resource: fixed_network} - fixed_subnet: {get_resource: fixed_subnet} - secgroup_base_id: {get_resource: secgroup_base} - secgroup_mesos_id: {get_resource: secgroup_mesos} - api_pool_id: {get_resource: api_pool} + fixed_network: "" + fixed_subnet: "" + secgroup_base_id: "" + secgroup_mesos_id: "" + api_pool_id: "" ###################################################################### # @@ -425,7 +315,7 @@ resources: mesos_slaves: type: OS::Heat::ResourceGroup depends_on: - - extrouter_inside + - mesos_masters properties: count: {get_param: number_of_slaves} removal_policies: [{resource_list: {get_param: slaves_to_remove}}] @@ -435,13 +325,13 @@ resources: ssh_key_name: {get_param: ssh_key_name} server_image: {get_param: server_image} slave_flavor: {get_param: slave_flavor} - fixed_network: {get_resource: fixed_network} - fixed_subnet: {get_resource: fixed_subnet} + fixed_network: "" + fixed_subnet: "" mesos_masters_ips: {list_join: [' ', {get_attr: [mesos_masters, mesos_master_ip]}]} external_network: {get_param: external_network} wait_condition_timeout: {get_param: wait_condition_timeout} executor_registration_timeout: {get_param: executor_registration_timeout} - secgroup_base_id: {get_resource: secgroup_base} + secgroup_base_id: "" http_proxy: {get_param: http_proxy} https_proxy: {get_param: https_proxy} no_proxy: {get_param: no_proxy} @@ -461,7 +351,7 @@ resources: outputs: api_address: - value: {get_attr: [api_address_switch, public_ip]} + value: {get_attr: [mesos_masters, resource.0.mesos_master_external_ip]} description: > This is the API endpoint of the Mesos master. Use this to access the Mesos API from outside the cluster. diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml index eee70e0ec..f7fc63620 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml @@ -59,45 +59,18 @@ resources: user_data_format: SOFTWARE_CONFIG metadata: cern-services: false - networks: - - port: {get_resource: mesos_master_eth0} - - mesos_master_eth0: - type: OS::Neutron::Port - properties: - network: {get_param: fixed_network} - security_groups: - - {get_param: secgroup_base_id} - - {get_param: secgroup_mesos_id} - fixed_ips: - - subnet: {get_param: fixed_subnet} - replacement_policy: AUTO - - mesos_master_floating: - type: OS::Neutron::FloatingIP - properties: - floating_network: {get_param: external_network} - port_id: {get_resource: mesos_master_eth0} - - api_pool_member: - type: Magnum::Optional::Neutron::LBaaS::PoolMember - properties: - pool: {get_param: api_pool_id} - address: {get_attr: [mesos_master_eth0, fixed_ips, 0, ip_address]} - subnet: { get_param: fixed_subnet } - protocol_port: 8080 outputs: mesos_master_ip: - value: {get_attr: [mesos_master_eth0, fixed_ips, 0, ip_address]} + value: {get_attr: [mesos-master, first_address]} description: > This is the "private" address of the Mesos master node. mesos_master_external_ip: - value: {get_attr: [mesos_master_floating, floating_ip_address]} + value: {get_attr: [mesos-master, first_address]} description: > This is the "public" address of the Mesos master node. mesos_server_id: - value: {get_resource: mesos_master} + value: {get_resource: mesos-master} description: > This is the logical id of the Mesos master node. diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml index 3e48ea60c..18932e6ff 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml @@ -143,14 +143,6 @@ resources: handle: {get_resource: slave_wait_handle} timeout: {get_param: wait_condition_timeout} - secgroup_all_open: - type: OS::Neutron::SecurityGroup - properties: - rules: - - protocol: icmp - - protocol: tcp - - protocol: udp - ###################################################################### # # software configs. these are components that are combined into @@ -245,33 +237,14 @@ resources: user_data: {get_resource: mesos_slave_init} metadata: cern-services: false - networks: - - port: {get_resource: mesos_slave_eth0} - - mesos_slave_eth0: - type: OS::Neutron::Port - properties: - network: {get_param: fixed_network} - security_groups: - - get_resource: secgroup_all_open - - get_param: secgroup_base_id - fixed_ips: - - subnet: {get_param: fixed_subnet} - replacement_policy: AUTO - - mesos_slave_floating: - type: OS::Neutron::FloatingIP - properties: - floating_network: {get_param: external_network} - port_id: {get_resource: mesos_slave_eth0} outputs: mesos_slave_ip: - value: {get_attr: [mesos_slave_eth0, fixed_ips, 0, ip_address]} + value: {get_attr: [mesos-slave, first_address]} description: > This is the "private" address of the Mesos slave node. mesos_slave_external_ip: - value: {get_attr: [mesos_slave_floating, floating_ip_address]} + value: {get_attr: [mesos-slave, first_address]} description: > This is the "public" address of the Mesos slave node. diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml index 4f893f614..d60a24d90 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml @@ -229,148 +229,6 @@ parameters: resources: - ###################################################################### - # - # network resources. allocate a network and router for our server. - # it would also be possible to take advantage of existing network - # resources (and have the deployer provide network and subnet ids, - # etc, as parameters), but I wanted to minmize the amount of - # configuration necessary to make this go. - - fixed_network: - type: "OS::Neutron::Net" - - # This is the subnet on which we will deploy our server. - fixed_subnet: - type: "OS::Neutron::Subnet" - properties: - cidr: {get_param: fixed_network_cidr} - network_id: - get_resource: fixed_network - dns_nameservers: - - get_param: dns_nameserver - - # create a router attached to the external network provided as a - # parameter to this stack. - extrouter: - type: "OS::Neutron::Router" - properties: - external_gateway_info: - network: - get_param: external_network - - # attached fixed_subnet to our extrouter router. - extrouter_inside: - type: "OS::Neutron::RouterInterface" - properties: - router_id: - get_resource: extrouter - subnet_id: - get_resource: - fixed_subnet - - ###################################################################### - # - # security groups. we need to permit network traffic of various - # sorts. - # - - secgroup_manager: - type: "OS::Neutron::SecurityGroup" - properties: - rules: - - protocol: icmp - - protocol: tcp - - protocol: udp - - ###################################################################### - # - # load balancers. - # - api_loadbalancer: - type: Magnum::Optional::Neutron::LBaaS::LoadBalancer - properties: - vip_subnet: {get_resource: fixed_subnet} - - api_listener: - type: Magnum::Optional::Neutron::LBaaS::Listener - properties: - loadbalancer: {get_resource: api_loadbalancer} - protocol: {get_param: loadbalancing_protocol} - protocol_port: {get_param: swarm_port} - - api_pool: - type: Magnum::Optional::Neutron::LBaaS::Pool - properties: - lb_algorithm: ROUND_ROBIN - listener: {get_resource: api_listener} - protocol: {get_param: loadbalancing_protocol} - - api_monitor: - type: Magnum::Optional::Neutron::LBaaS::HealthMonitor - properties: - type: TCP - delay: 5 - max_retries: 5 - timeout: 5 - pool: { get_resource: api_pool } - - api_pool_floating: - type: Magnum::Optional::Neutron::FloatingIP - depends_on: - - extrouter_inside - properties: - floating_network: {get_param: external_network} - port_id: {get_attr: [api_loadbalancer, vip_port_id]} - - etcd_loadbalancer: - type: Magnum::Optional::Neutron::LBaaS::LoadBalancer - properties: - vip_subnet: {get_resource: fixed_subnet} - - etcd_listener: - type: Magnum::Optional::Neutron::LBaaS::Listener - properties: - loadbalancer: {get_resource: etcd_loadbalancer} - protocol: HTTP - protocol_port: 2379 - - etcd_pool: - type: Magnum::Optional::Neutron::LBaaS::Pool - properties: - lb_algorithm: ROUND_ROBIN - listener: {get_resource: etcd_listener} - protocol: HTTP - - etcd_monitor: - type: Magnum::Optional::Neutron::LBaaS::HealthMonitor - properties: - type: TCP - delay: 5 - max_retries: 5 - timeout: 5 - pool: { get_resource: etcd_pool } - - ###################################################################### - # - # resources that expose the IPs of either the swarm master or a given - # LBaaS pool depending on whether LBaaS is enabled for the cluster. - # - - api_address_switch: - type: Magnum::ApiGatewaySwitcher - properties: - pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]} - pool_private_ip: {get_attr: [api_loadbalancer, vip_address]} - master_public_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]} - master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]} - - etcd_address_switch: - type: Magnum::ApiGatewaySwitcher - properties: - pool_private_ip: {get_attr: [etcd_loadbalancer, vip_address]} - master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]} - ###################################################################### # # Swarm manager is responsible for the entire cluster and manages the @@ -380,8 +238,6 @@ resources: swarm_masters: type: "OS::Heat::ResourceGroup" - depends_on: - - extrouter_inside properties: count: {get_param: number_of_masters} resource_def: @@ -392,27 +248,27 @@ resources: server_flavor: {get_param: master_flavor} docker_volume_size: {get_param: docker_volume_size} docker_storage_driver: {get_param: docker_storage_driver} - fixed_network_id: {get_resource: fixed_network} - fixed_subnet_id: {get_resource: fixed_subnet} + fixed_network_id: "" + fixed_subnet_id: "" external_network: {get_param: external_network} discovery_url: {get_param: discovery_url} http_proxy: {get_param: http_proxy} https_proxy: {get_param: https_proxy} no_proxy: {get_param: no_proxy} - swarm_api_ip: {get_attr: [api_loadbalancer, vip_address]} + swarm_api_ip: "" cluster_uuid: {get_param: cluster_uuid} magnum_url: {get_param: magnum_url} tls_disabled: {get_param: tls_disabled} - secgroup_swarm_master_id: {get_resource: secgroup_manager} + secgroup_swarm_master_id: "" network_driver: {get_param: network_driver} flannel_network_cidr: {get_param: flannel_network_cidr} flannel_network_subnetlen: {get_param: flannel_network_subnetlen} flannel_backend: {get_param: flannel_backend} swarm_port: {get_param: swarm_port} - api_pool_id: {get_resource: api_pool} - etcd_pool_id: {get_resource: etcd_pool} - etcd_server_ip: {get_attr: [etcd_loadbalancer, vip_address]} - api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]} + api_pool_id: "" + etcd_pool_id: "" + etcd_server_ip: "" + api_ip_address: "" swarm_version: {get_param: swarm_version} trustee_user_id: {get_param: trustee_user_id} trustee_password: {get_param: trustee_password} @@ -424,7 +280,7 @@ resources: swarm_nodes: type: "OS::Heat::ResourceGroup" depends_on: - - extrouter_inside + - swarm_masters properties: count: {get_param: number_of_nodes} resource_def: @@ -435,21 +291,21 @@ resources: server_flavor: {get_param: node_flavor} docker_volume_size: {get_param: docker_volume_size} docker_storage_driver: {get_param: docker_storage_driver} - fixed_network_id: {get_resource: fixed_network} - fixed_subnet_id: {get_resource: fixed_subnet} + fixed_network_id: "" + fixed_subnet_id: "" external_network: {get_param: external_network} http_proxy: {get_param: http_proxy} https_proxy: {get_param: https_proxy} no_proxy: {get_param: no_proxy} - swarm_api_ip: {get_attr: [api_address_switch, private_ip]} + swarm_api_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]} cluster_uuid: {get_param: cluster_uuid} magnum_url: {get_param: magnum_url} tls_disabled: {get_param: tls_disabled} - secgroup_swarm_node_id: {get_resource: secgroup_manager} + secgroup_swarm_node_id: "" flannel_network_cidr: {get_param: flannel_network_cidr} network_driver: {get_param: network_driver} - etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]} - api_ip_address: {get_attr: [api_address_switch, public_ip]} + etcd_server_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]} + api_ip_address: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]} swarm_version: {get_param: swarm_version} trustee_domain_id: {get_param: trustee_domain_id} trustee_user_id: {get_param: trustee_user_id} @@ -469,31 +325,17 @@ resources: outputs: api_address: - value: - str_replace: - template: api_ip_address - params: - api_ip_address: {get_attr: [api_address_switch, public_ip]} + value: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]} description: > This is the API endpoint of the Swarm masters. Use this to access the Swarm API server from outside the cluster. - swarm_masters_private: - value: {get_attr: [swarm_masters, swarm_master_ip]} - description: > - This is a list of the "private" addresses of all the Swarm masters. - swarm_masters: value: {get_attr: [swarm_masters, swarm_master_external_ip]} description: > This is a list of "public" ip addresses of all Swarm masters. Use these addresses to log into the Swarm masters via ssh. - swarm_nodes_private: - value: {get_attr: [swarm_nodes, swarm_node_ip]} - description: > - This is a list of the "private" addresses of all the Swarm nodes. - swarm_nodes: value: {get_attr: [swarm_nodes, swarm_node_external_ip]} description: > diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/make-cert.py b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/make-cert.py index c57848c1a..44d729451 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/make-cert.py +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/make-cert.py @@ -16,6 +16,7 @@ import json import os +import socket import subprocess import requests @@ -66,15 +67,14 @@ def create_dirs(): def _get_public_ip(): - return requests.get(PUBLIC_IP_URL).text + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(('8.8.8.8', 0)) # connecting to a UDP address doesn't send packets + return s.getsockname()[0] def _build_subject_alt_names(config): subject_alt_names = [ 'IP:%s' % _get_public_ip(), - 'IP:%s' % config['API_IP_ADDRESS'], - 'IP:%s' % config['SWARM_NODE_IP'], - 'IP:%s' % config['SWARM_API_IP'], 'IP:127.0.0.1' ] return ','.join(subject_alt_names) diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.sh new file mode 100644 index 000000000..5088f8626 --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.sh @@ -0,0 +1,39 @@ +#!/bin/sh +SWARM_API_IP_TMP=$SWARM_API_IP +SWARM_NODE_IP_TMP=$SWARM_NODE_IP +API_IP_ADDRESS_TMP=$API_IP_ADDRESS +ETCD_SERVER_IP_TMP=$ETCD_SERVER_IP + +cat > /etc/sysconfig/heat-params << END_SERVICE +WAIT_HANDLE_ENDPOINT="$WAIT_HANDLE_ENDPOINT" +WAIT_HANDLE_TOKEN="$WAIT_HANDLE_TOKEN" +WAIT_CURL="$WAIT_CURL" +ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL" +DOCKER_VOLUME="$DOCKER_VOLUME" +DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" +HTTP_PROXY="$HTTP_PROXY" +HTTPS_PROXY="$HTTPS_PROXY" +NO_PROXY="$NO_PROXY" +SWARM_API_IP="${SWARM_API_IP_TMP:-$(hostname -I | cut -d' ' -f1)}" +SWARM_NODE_IP="${SWARM_NODE_IP_TMP:-$(hostname -I | cut -d' ' -f1)}" +CLUSTER_UUID="$CLUSTER_UUID" +MAGNUM_URL="$MAGNUM_URL" +TLS_DISABLED="$TLS_DISABLED" +NETWORK_DRIVER="$NETWORK_DRIVER" +FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR" +FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN" +FLANNEL_BACKEND="$FLANNEL_BACKEND" +ETCD_SERVER_IP="${ETCD_SERVER_IP_TMP:-$(hostname -I | cut -d' ' -f1)}" +API_IP_ADDRESS="${API_IP_ADDRESS_TMP:-$(hostname -I | cut -d' ' -f1)}" +SWARM_VERSION="$SWARM_VERSION" +TRUSTEE_USER_ID="$TRUSTEE_USER_ID" +TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD" +TRUST_ID="$TRUST_ID" +AUTH_URL="$AUTH_URL" +VOLUME_DRIVER="$VOLUME_DRIVER" +REXRAY_PREEMPT="$REXRAY_PREEMPT" +END_SERVICE + +chown root:root /etc/sysconfig/heat-params +chmod 644 /etc/sysconfig/heat-params + diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.sh new file mode 100644 index 000000000..872ed7ea6 --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.sh @@ -0,0 +1,39 @@ +#!/bin/sh +SWARM_API_IP_TMP=$SWARM_API_IP +SWARM_NODE_IP_TMP=$SWARM_NODE_IP +API_IP_ADDRESS_TMP=$API_IP_ADDRESS +ETCD_SERVER_IP_TMP=$ETCD_SERVER_IP + +cat > /etc/sysconfig/heat-params << END_SERVICE +WAIT_HANDLE_ENDPOINT="$WAIT_HANDLE_ENDPOINT" +WAIT_HANDLE_TOKEN="$WAIT_HANDLE_TOKEN" +WAIT_CURL="$WAIT_CURL" +DOCKER_VOLUME="$DOCKER_VOLUME" +DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" +HTTP_PROXY="$HTTP_PROXY" +HTTPS_PROXY="$HTTPS_PROXY" +NO_PROXY="$NO_PROXY" +SWARM_API_IP="${SWARM_API_IP_TMP:-$(hostname -I | cut -d' ' -f1)}" +SWARM_NODE_IP="${SWARM_NODE_IP_TMP:-$(hostname -I | cut -d' ' -f1)}" +CLUSTER_UUID="$CLUSTER_UUID" +MAGNUM_URL="$MAGNUM_URL" +TLS_DISABLED="$TLS_DISABLED" +NETWORK_DRIVER="$NETWORK_DRIVER" +ETCD_SERVER_IP="${ETCD_SERVER_IP_TMP:-$(hostname -I | cut -d' ' -f1)}" +API_IP_ADDRESS="${API_IP_ADDRESS_TMP:-$(hostname -I | cut -d' ' -f1)}" +SWARM_VERSION="$SWARM_VERSION" +TRUSTEE_DOMAIN_ID="$TRUSTEE_DOMAIN_ID" +TRUSTEE_USER_ID="$TRUSTEE_USER_ID" +TRUSTEE_USERNAME="$TRUSTEE_USERNAME" +TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD" +TRUST_ID="$TRUST_ID" +AUTH_URL="$AUTH_URL" +REGISTRY_ENABLED="$REGISTRY_ENABLED" +REGISTRY_PORT="$REGISTRY_PORT" +SWIFT_REGION="$SWIFT_REGION" +REGISTRY_CONTAINER="$REGISTRY_CONTAINER" +REGISTRY_INSECURE="$REGISTRY_INSECURE" +REGISTRY_CHUNKSIZE="$REGISTRY_CHUNKSIZE" +VOLUME_DRIVER="$VOLUME_DRIVER" +REXRAY_PREEMPT="$REXRAY_PREEMPT" +END_SERVICE diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-swarm-master-service.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-swarm-master-service.sh index 2da702c0f..c9c203870 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-swarm-master-service.sh +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-swarm-master-service.sh @@ -1,5 +1,7 @@ #!/bin/sh +. /etc/sysconfig/heat-params + cat > /etc/systemd/system/swarm-manager.service << END_SERVICE_TOP [Unit] Description=Swarm Manager @@ -21,7 +23,7 @@ ExecStart=/usr/bin/docker run --name swarm-manager \\ swarm:$SWARM_VERSION \\ manage -H tcp://0.0.0.0:2375 \\ --replication \\ - --advertise $NODE_IP:2376 \\ + --advertise $SWARM_NODE_IP:2376 \\ END_SERVICE_TOP if [ $TLS_DISABLED = 'False' ]; then diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index bafc7d32f..70fcb247a 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -167,26 +167,6 @@ resources: handle: {get_resource: master_wait_handle} timeout: 6000 - ###################################################################### - # - # resource that exposes the IPs of either the Swarm master or the API - # LBaaS pool depending on whether LBaaS is enabled for the cluster. - # - - api_address_switch: - type: Magnum::ApiGatewaySwitcher - properties: - pool_public_ip: {get_param: api_ip_address} - pool_private_ip: {get_param: swarm_api_ip} - master_public_ip: {get_attr: [swarm_master_floating, floating_ip_address]} - master_private_ip: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} - - etcd_address_switch: - type: Magnum::ApiGatewaySwitcher - properties: - pool_private_ip: {get_param: etcd_server_ip} - master_private_ip: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} - ###################################################################### # # software configs. these are components that are combined into @@ -198,7 +178,7 @@ resources: group: ungrouped config: str_replace: - template: {get_file: fragments/write-heat-params-master.yaml} + template: {get_file: fragments/write-heat-params-master.sh} params: "$WAIT_HANDLE_ENDPOINT": {get_attr: [master_wait_handle, endpoint]} "$WAIT_HANDLE_TOKEN": {get_attr: [master_wait_handle, token]} @@ -210,8 +190,8 @@ resources: "$HTTP_PROXY": {get_param: http_proxy} "$HTTPS_PROXY": {get_param: https_proxy} "$NO_PROXY": {get_param: no_proxy} - "$SWARM_API_IP": {get_attr: [api_address_switch, private_ip]} - "$SWARM_NODE_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} + "$SWARM_API_IP": {get_param: swarm_api_ip} + "$SWARM_NODE_IP": "" "$CLUSTER_UUID": {get_param: cluster_uuid} "$MAGNUM_URL": {get_param: magnum_url} "$TLS_DISABLED": {get_param: tls_disabled} @@ -219,8 +199,8 @@ resources: "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} "$FLANNEL_BACKEND": {get_param: flannel_backend} - "$ETCD_SERVER_IP": {get_attr: [etcd_address_switch, private_ip]} - "$API_IP_ADDRESS": {get_attr: [api_address_switch, public_ip]} + "$ETCD_SERVER_IP": "" + "$API_IP_ADDRESS": "" "$SWARM_VERSION": {get_param: swarm_version} "$TRUSTEE_USER_ID": {get_param: trustee_user_id} "$TRUSTEE_PASSWORD": {get_param: trustee_password} @@ -307,15 +287,9 @@ resources: str_replace: template: {get_file: fragments/write-swarm-master-service.sh} params: - "$ETCD_SERVER_IP": {get_attr: [etcd_address_switch, private_ip]} - "$NODE_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} "$WAIT_HANDLE_ENDPOINT": {get_attr: [master_wait_handle, endpoint]} "$WAIT_HANDLE_TOKEN": {get_attr: [master_wait_handle, token]} - "$HTTP_PROXY": {get_param: http_proxy} - "$HTTPS_PROXY": {get_param: https_proxy} - "$NO_PROXY": {get_param: no_proxy} - "$TLS_DISABLED": {get_param: tls_disabled} - "$SWARM_VERSION": {get_param: swarm_version} + # only replace $WAIT_HANDLE, others are sourced from heat-params enable_services: type: "OS::Heat::SoftwareConfig" @@ -399,46 +373,6 @@ resources: user_data: {get_resource: swarm_master_init} metadata: cern-services: false - networks: - - port: - get_resource: swarm_master_eth0 - - swarm_master_eth0: - type: "OS::Neutron::Port" - properties: - network_id: - get_param: fixed_network_id - security_groups: - - {get_param: secgroup_swarm_master_id} - fixed_ips: - - subnet_id: - get_param: fixed_subnet_id - allowed_address_pairs: - - ip_address: {get_param: flannel_network_cidr} - - swarm_master_floating: - type: "OS::Neutron::FloatingIP" - properties: - floating_network: - get_param: external_network - port_id: - get_resource: swarm_master_eth0 - - api_pool_member: - type: Magnum::Optional::Neutron::LBaaS::PoolMember - properties: - pool: {get_param: api_pool_id} - address: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} - subnet: { get_param: fixed_subnet_id } - protocol_port: {get_param: swarm_port} - - etcd_pool_member: - type: Magnum::Optional::Neutron::LBaaS::PoolMember - properties: - pool: {get_param: etcd_pool_id} - address: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} - subnet: { get_param: fixed_subnet_id } - protocol_port: 2379 ###################################################################### # @@ -461,11 +395,11 @@ resources: outputs: swarm_master_ip: - value: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} + value: {get_attr: [swarm-master, first_address]} description: > This is the "private" addresses of all the Swarm master. swarm_master_external_ip: - value: {get_attr: [swarm_master_floating, floating_ip_address]} + value: {get_attr: [swarm-master, first_address]} description: > This is the "public" ip addresses of Swarm master. diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index 4bec5793d..ea61a069f 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -183,7 +183,7 @@ resources: group: ungrouped config: str_replace: - template: {get_file: fragments/write-heat-params-node.yaml} + template: {get_file: fragments/write-heat-params-node.sh} params: "$WAIT_HANDLE_ENDPOINT": {get_attr: [node_wait_handle, endpoint]} "$WAIT_HANDLE_TOKEN": {get_attr: [node_wait_handle, token]} @@ -195,7 +195,7 @@ resources: "$HTTPS_PROXY": {get_param: https_proxy} "$NO_PROXY": {get_param: no_proxy} "$SWARM_API_IP": {get_param: swarm_api_ip} - "$SWARM_NODE_IP": {get_attr: [swarm_node_eth0, fixed_ips, 0, ip_address]} + "$SWARM_NODE_IP": "" "$CLUSTER_UUID": {get_param: cluster_uuid} "$MAGNUM_URL": {get_param: magnum_url} "$TLS_DISABLED": {get_param: tls_disabled} @@ -363,30 +363,6 @@ resources: user_data: {get_resource: swarm_node_init} metadata: cern-services: false - networks: - - port: - get_resource: swarm_node_eth0 - - swarm_node_eth0: - type: "OS::Neutron::Port" - properties: - network_id: - get_param: fixed_network_id - security_groups: - - {get_param: secgroup_swarm_node_id} - fixed_ips: - - subnet_id: - get_param: fixed_subnet_id - allowed_address_pairs: - - ip_address: {get_param: flannel_network_cidr} - - swarm_node_floating: - type: "OS::Neutron::FloatingIP" - properties: - floating_network: - get_param: external_network - port_id: - get_resource: swarm_node_eth0 ###################################################################### # @@ -409,11 +385,11 @@ resources: outputs: swarm_node_ip: - value: {get_attr: [swarm_node_eth0, fixed_ips, 0, ip_address]} + value: {get_attr: [swarm-node, first_address]} description: > This is the "private" address of the Swarm node. swarm_node_external_ip: - value: {get_attr: [swarm_node_floating, floating_ip_address]} + value: {get_attr: [swarm-node, first_address]} description: > This is the "public" address of the Swarm node. -- GitLab From aec3c023d6e616b963bd01a66eb7b24b67cb265a Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Thu, 15 Dec 2016 17:13:36 +0100 Subject: [PATCH 9/9] [cern] upgrade swarm to version 1.2.5 Change-Id: I63098bc0e16e4c75961d0078345007aabc31db61 --- magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml index d60a24d90..3d1c75be3 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml @@ -145,7 +145,7 @@ parameters: swarm_version: type: string description: version of swarm used for swarm cluster - default: 1.0.0 + default: 1.2.5 trustee_domain_id: type: string -- GitLab