From 258b9521517dc226b7423c11596ca2e412b6d9c3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 14 Aug 2020 15:05:59 +0530 Subject: [PATCH 01/27] wip: move params to test_data Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 66 ++-- .../test_kubernetes_supported_versions.py | 16 +- tools/marvin/marvin/config/test_data.py | 286 +++++++++++------- 3 files changed, 207 insertions(+), 161 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 492c9702a9d5..06147dd961b5 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -72,57 +72,41 @@ def setUpClass(cls): cls.kubernetes_version_ids = [] if cls.setup_failed == False: try: - cls.kubernetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://download.cloudstack.org/cks/setup-1.14.9.iso') + cls.kubernetes_version_1 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.14.9"]) cls.kubernetes_version_ids.append(cls.kubernetes_version_1.id) except Exception as e: cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, http://download.cloudstack.org/cks/setup-1.14.9.iso, %s" % e) + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.14.9"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.14.9"]["url"], e)) if cls.setup_failed == False: try: - cls.kubernetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://download.cloudstack.org/cks/setup-1.15.0.iso') + cls.kubernetes_version_2 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.15.0"]) cls.kubernetes_version_ids.append(cls.kubernetes_version_2.id) except Exception as e: cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, http://download.cloudstack.org/cks/setup-1.15.0.iso, %s" % e) + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.15.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.15.0"]["url"], e)) if cls.setup_failed == False: try: - cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion('1.16.0', 'http://download.cloudstack.org/cks/setup-1.16.0.iso') + cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.0"]) cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) except Exception as e: cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, http://download.cloudstack.org/cks/setup-1.16.0.iso, %s" % e) + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.16.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.0"]["url"], e)) if cls.setup_failed == False: try: - cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion('1.16.3', 'http://download.cloudstack.org/cks/setup-1.16.3.iso') + cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.3"]) cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) except Exception as e: cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, http://download.cloudstack.org/cks/setup-1.16.3.iso, %s" % e) - - cks_template_data = { - "name": "Kubernetes-Service-Template", - "displaytext": "Kubernetes-Service-Template", - "format": "qcow2", - "hypervisor": "kvm", - "ostype": "CoreOS", - "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2", - "ispublic": "True", - "isextractable": "True" - } - cks_template_data_details = [] - if cls.hypervisor.lower() == "vmware": - cks_template_data["url"] = "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova" - cks_template_data["format"] = "OVA" - cks_template_data_details = [{"keyboard":"us","nicAdapter":"Vmxnet3","rootDiskController":"pvscsi"}] - elif cls.hypervisor.lower() == "xenserver": - cks_template_data["url"] = "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2" - cks_template_data["format"] = "VHD" - elif cls.hypervisor.lower() == "kvm": - cks_template_data["requireshvm"] = "True" + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) + if cls.setup_failed == False: cls.cks_template = Template.register( cls.apiclient, - cks_template_data, + cls.services["cks_templates"][cls.hypervisor.lower()], zoneid=cls.zone.id, hypervisor=cls.hypervisor, details=cks_template_data_details @@ -140,14 +124,8 @@ def setUpClass(cls): cls.cks_template_name_key, cls.cks_template.name) - cks_offering_data = { - "name": "CKS-Instance", - "displaytext": "CKS Instance", - "cpunumber": 2, - "cpuspeed": 1000, - "memory": 2048, - } - cks_offering_data["name"] = cks_offering_data["name"] + '-' + random_gen() + cks_offering_data = cls.services["cks_service_offering"] + cks_offering_data["name"] = 'CKS-Instance-' + random_gen() if cls.setup_failed == False: cls.cks_service_offering = ServiceOffering.create( cls.apiclient, @@ -279,13 +257,13 @@ def listKubernetesSupportedVersion(cls, version_id): return versionResponse[0] @classmethod - def addKubernetesSupportedVersion(cls, semantic_version, iso_url): + def addKubernetesSupportedVersion(cls, version_service): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() - addKubernetesSupportedVersionCmd.semanticversion = semantic_version - addKubernetesSupportedVersionCmd.name = 'v' + semantic_version + '-' + random_gen() - addKubernetesSupportedVersionCmd.url = iso_url - addKubernetesSupportedVersionCmd.mincpunumber = 2 - addKubernetesSupportedVersionCmd.minmemory = 2048 + addKubernetesSupportedVersionCmd.semanticversion = version_service["version_service"] + addKubernetesSupportedVersionCmd.name = 'v' + version_service["version_service"] + '-' + random_gen() + addKubernetesSupportedVersionCmd.url = version_service["url"] + addKubernetesSupportedVersionCmd.mincpunumber = version_service["mincpunumber"] + addKubernetesSupportedVersionCmd.minmemory = version_service["minmemory"] kubernetes_version = cls.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) cls.debug("Waiting for Kubernetes version with ID %s to be ready" % kubernetes_version.id) cls.waitForKubernetesSupportedVersionIsoReadyState(kubernetes_version.id) diff --git a/test/integration/smoke/test_kubernetes_supported_versions.py b/test/integration/smoke/test_kubernetes_supported_versions.py index b2202053ebfc..90f9e28c3c12 100644 --- a/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/test/integration/smoke/test_kubernetes_supported_versions.py @@ -130,12 +130,12 @@ def test_01_add_delete_kubernetes_supported_version(self): # 2. The Cloud Database contains the valid information when listKubernetesSupportedVersions is called """ - version = '1.16.3' - name = 'v' + version + '-' + random_gen() + version = self.services["cks_kubernetes_version"]["1.16.3"] + name = 'v' + version["semanticversion"] + '-' + random_gen() self.debug("Adding Kubernetes supported version with name: %s" % name) - version_response = self.addKubernetesSupportedVersion(version, name, self.zone.id, self.kubernetes_version_iso_url) + version_response = self.addKubernetesSupportedVersion(version["semanticversion"], name, self.zone.id, version["url"], version["mincpunumber"], version["mincpumemory"]) list_versions_response = self.listKubernetesSupportedVersion(version_response.id) @@ -147,8 +147,8 @@ def test_01_add_delete_kubernetes_supported_version(self): self.assertEqual( list_versions_response.semanticversion, - version, - "Check KubernetesSupportedVersion version {}, {}".format(list_versions_response.semanticversion, version) + version["semanticversion"], + "Check KubernetesSupportedVersion version {}, {}".format(list_versions_response.semanticversion, version["semanticversion"]) ) self.assertEqual( list_versions_response.zoneid, @@ -228,14 +228,14 @@ def test_03_add_invalid_kubernetes_supported_version(self): self.debug("Unsupported version error check successful, API failure: %s" % e) return - def addKubernetesSupportedVersion(self, version, name, zoneId, isoUrl): + def addKubernetesSupportedVersion(self, version, name, zoneId, isoUrl, mincpunumber=2, minmemory=2048): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() addKubernetesSupportedVersionCmd.semanticversion = version addKubernetesSupportedVersionCmd.name = name addKubernetesSupportedVersionCmd.zoneid = zoneId addKubernetesSupportedVersionCmd.url = isoUrl - addKubernetesSupportedVersionCmd.mincpunumber = 2 - addKubernetesSupportedVersionCmd.minmemory = 2048 + addKubernetesSupportedVersionCmd.mincpunumber = mincpunumber + addKubernetesSupportedVersionCmd.minmemory = minmemory versionResponse = self.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) if not versionResponse: self.cleanup.append(versionResponse) diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 929741b7b0d1..28c53fe4a258 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1847,119 +1847,187 @@ }, }, "configurableData": - { - "portableIpRange": { - "gateway": "10.223.59.1", - "netmask": "255.255.255.0", - "startip": "10.223.59.200", - "endip": "10.223.59.240", - "vlan": "1000" - }, - "netscaler": { - "ipaddress": "", - "username": "", - "password": "", - "networkdevicetype": "", - "publicinterface": "", - "privateinterface": "", - "numretries": "", - "lbdevicededicated": "False", - "lbdevicecapacity": 2, - "port": 22 - }, - "iscsi": { - "url": "", - "name": "Primary iSCSI" - }, - "host": { - "publicport": 22, - "username": "root", - "password": "password", - }, - "ldap_account": { - "email": "", - "firstname": "", - "lastname": "", - "username": "", - "password": "", - }, - "link_ldap_details": { - "domain_name": "", - "accounttype": "", - "name": "", - "type": "", - "admin": "", - "linkLdapUsername": "", - "linkLdapPassword": "", - "linkLdapNestedUser": "", - "linkLdapNestedPassword": "" + { + "portableIpRange": { + "gateway": "10.223.59.1", + "netmask": "255.255.255.0", + "startip": "10.223.59.200", + "endip": "10.223.59.240", + "vlan": "1000" + }, + "netscaler": { + "ipaddress": "", + "username": "", + "password": "", + "networkdevicetype": "", + "publicinterface": "", + "privateinterface": "", + "numretries": "", + "lbdevicededicated": "False", + "lbdevicecapacity": 2, + "port": 22 + }, + "iscsi": { + "url": "", + "name": "Primary iSCSI" + }, + "host": { + "publicport": 22, + "username": "root", + "password": "password", + }, + "ldap_account": { + "email": "", + "firstname": "", + "lastname": "", + "username": "", + "password": "", + }, + "link_ldap_details": { + "domain_name": "", + "accounttype": "", + "name": "", + "type": "", + "admin": "", + "linkLdapUsername": "", + "linkLdapPassword": "", + "linkLdapNestedUser": "", + "linkLdapNestedPassword": "" + }, + "ldap_configuration": { + "basedn": "", + "emailAttribute": "", + "userObject": "", + "usernameAttribute": "", + "hostname": "", + "port": "", + "ldapUsername": "", + "ldapPassword": "" + }, + "systemVmDelay": 120, + "setUsageConfigurationThroughTestCase": True, + "vmware_cluster": { + "hypervisor": 'VMware', + "clustertype": 'ExternalManaged', + "username": '', + "password": '', + "url": '', + "clustername": 'VMWare Cluster with Space in DC name', + }, + "upload_volume": { + "diskname": "UploadVol", + "format": "VHD", + "url": "http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.vhd.bz2", + "checksum": "", + }, + "bootableIso": + { + "displaytext": "Test Bootable ISO", + "name": "testISO", + "bootable": True, + "ispublic": False, + "url": "http://dl.openvm.eu/cloudstack/iso/TinyCore-8.0.iso", + "ostype": 'Other Linux (64-bit)', + "mode": 'HTTP_DOWNLOAD' }, - "ldap_configuration": { - "basedn": "", - "emailAttribute": "", - "userObject": "", - "usernameAttribute": "", - "hostname": "", - "port": "", - "ldapUsername": "", - "ldapPassword": "" + "setHostConfigurationForIngressRule": False, + "restartManagementServerThroughTestCase": False, + "vmxnet3template": { + "displaytext": "VMXNET3 Template", + "name": "VMXNET3 template", + "ostype": "CentOS 5.6 (64-bit)", + "isfeatured": True, + "ispublic": False, + "isextractable": True, + "mode": "HTTP_DOWNLOAD", + "templatefilter": "self", + "url": "http://people.apache.org/~sanjeev/systemvm64template-2014-09-30-4.3-vmware.ova", + "hypervisor": "vmware", + "format": "OVA", + "nicadapter": "vmxnet3", + "kvm": { + "url": "" }, - "systemVmDelay": 120, - "setUsageConfigurationThroughTestCase": True, - "vmware_cluster": { - "hypervisor": 'VMware', - "clustertype": 'ExternalManaged', - "username": '', - "password": '', - "url": '', - "clustername": 'VMWare Cluster with Space in DC name', + "vmware": { + "url": "" }, - "upload_volume": { - "diskname": "UploadVol", - "format": "VHD", - "url": "http://download.cloudstack.org/releases/2.0.0/UbuntuServer-10-04-64bit.vhd.bz2", - "checksum": "", + "xenserver": { + "url": "" }, - "bootableIso": - { - "displaytext": "Test Bootable ISO", - "name": "testISO", - "bootable": True, - "ispublic": False, - "url": "http://dl.openvm.eu/cloudstack/iso/TinyCore-8.0.iso", - "ostype": 'Other Linux (64-bit)', - "mode": 'HTTP_DOWNLOAD' - }, - "setHostConfigurationForIngressRule": False, - "restartManagementServerThroughTestCase": False, - "vmxnet3template": { - "displaytext": "VMXNET3 Template", - "name": "VMXNET3 template", - "ostype": "CentOS 5.6 (64-bit)", - "isfeatured": True, - "ispublic": False, - "isextractable": True, - "mode": "HTTP_DOWNLOAD", - "templatefilter": "self", - "url": "http://people.apache.org/~sanjeev/systemvm64template-2014-09-30-4.3-vmware.ova", - "hypervisor": "vmware", - "format": "OVA", - "nicadapter": "vmxnet3", - "kvm": { - "url": "" - }, - "vmware": { - "url": "" - }, - "xenserver": { - "url": "" - }, - "hyperv": { - "url": "" - }, - "ostype": 'CentOS 5.3 (64-bit)', - "mode": 'HTTP_DOWNLOAD' - } + "hyperv": { + "url": "" + }, + "ostype": 'CentOS 5.3 (64-bit)', + "mode": 'HTTP_DOWNLOAD' + } + }, + "cks_kubernetes_versions": { + "1.14.9": { + "semanticversion": "1.14.9", + "url": "http://download.cloudstack.org/cks/setup-1.14.9.iso", + "mincpunumber": 2, + "minmemory": 2048 + }, + "1.15.0": { + "semanticversion": "1.15.0", + "url": "http://download.cloudstack.org/cks/setup-1.15.0.iso", + "mincpunumber": 2, + "minmemory": 2048 + }, + "1.16.0": { + "semanticversion": "1.16.0", + "url": "http://download.cloudstack.org/cks/setup-1.16.0.iso", + "mincpunumber": 2, + "minmemory": 2048 + }, + "1.16.3": { + "semanticversion": "1.16.3", + "url": "http://download.cloudstack.org/cks/setup-1.16.3.iso", + "mincpunumber": 2, + "minmemory": 2048 } + }, + "cks_templates": { + "kvm": { + "name": "Kubernetes-Service-Template-kvm", + "displaytext": "Kubernetes-Service-Template kvm", + "format": "qcow2", + "hypervisor": "kvm", + "ostype": "CoreOS", + "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2"", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True" + }, + "xenserver": { + "name": "Kubernetes-Service-Template-xen", + "displaytext": "Kubernetes-Service-Template xen", + "format": "vhd", + "hypervisor": "xenserver", + "ostype": "CoreOS", + "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True" + }, + "vmware": { + "name": "Kubernetes-Service-Template-vmware", + "displaytext": "Kubernetes-Service-Template vmware", + "format": "ova", + "hypervisor": "vmware", + "ostype": "CoreOS", + "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova", + "requireshvm": "True", + "ispublic": "True", + "details": [{"keyboard":"us","nicAdapter":"Vmxnet3","rootDiskController":"pvscsi"}] + } + }, + "cks_compute_offering": { + "name": "CKS-Instance", + "displaytext": "CKS Instance", + "cpunumber": 2, + "cpuspeed": 1000, + "memory": 2048 + } } From 53c9e48819cb8788009d52e01b4c6f135848ca0a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 21 Aug 2020 12:43:03 +0530 Subject: [PATCH 02/27] fix Signed-off-by: Abhishek Kumar --- tools/marvin/marvin/config/test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 28c53fe4a258..89efbebf025b 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1995,7 +1995,7 @@ "format": "qcow2", "hypervisor": "kvm", "ostype": "CoreOS", - "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2"", + "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2", "requireshvm": "True", "ispublic": "True", "isextractable": "True" From 03ac47ac02b4d70dba53857345d456677bee35d1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 21 Aug 2020 13:19:14 +0530 Subject: [PATCH 03/27] fix version test Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_supported_versions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_supported_versions.py b/test/integration/smoke/test_kubernetes_supported_versions.py index 90f9e28c3c12..ffeb00599d4d 100644 --- a/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/test/integration/smoke/test_kubernetes_supported_versions.py @@ -130,12 +130,12 @@ def test_01_add_delete_kubernetes_supported_version(self): # 2. The Cloud Database contains the valid information when listKubernetesSupportedVersions is called """ - version = self.services["cks_kubernetes_version"]["1.16.3"] + version = self.services["cks_kubernetes_versions"]["1.16.3"] name = 'v' + version["semanticversion"] + '-' + random_gen() self.debug("Adding Kubernetes supported version with name: %s" % name) - version_response = self.addKubernetesSupportedVersion(version["semanticversion"], name, self.zone.id, version["url"], version["mincpunumber"], version["mincpumemory"]) + version_response = self.addKubernetesSupportedVersion(version["semanticversion"], name, self.zone.id, version["url"], version["mincpunumber"], version["minmemory"]) list_versions_response = self.listKubernetesSupportedVersion(version_response.id) From 13f9b0c65f0b54d60178f5ce4c3c10729336a5a9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 21 Aug 2020 13:59:08 +0530 Subject: [PATCH 04/27] fix cluster test Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 4 ++-- tools/marvin/marvin/config/test_data.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 06147dd961b5..ff864f58171e 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -259,8 +259,8 @@ def listKubernetesSupportedVersion(cls, version_id): @classmethod def addKubernetesSupportedVersion(cls, version_service): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() - addKubernetesSupportedVersionCmd.semanticversion = version_service["version_service"] - addKubernetesSupportedVersionCmd.name = 'v' + version_service["version_service"] + '-' + random_gen() + addKubernetesSupportedVersionCmd.semanticversion = version_service["semanticversion"] + addKubernetesSupportedVersionCmd.name = 'v' + version_service["semanticversion"] + '-' + random_gen() addKubernetesSupportedVersionCmd.url = version_service["url"] addKubernetesSupportedVersionCmd.mincpunumber = version_service["mincpunumber"] addKubernetesSupportedVersionCmd.minmemory = version_service["minmemory"] diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 89efbebf025b..47087176be2d 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -2023,7 +2023,7 @@ "details": [{"keyboard":"us","nicAdapter":"Vmxnet3","rootDiskController":"pvscsi"}] } }, - "cks_compute_offering": { + "cks_service_offering": { "name": "CKS-Instance", "displaytext": "CKS Instance", "cpunumber": 2, From 732f8d3ddd287c00822a29450e4c7fea6d2483c3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 27 Aug 2020 14:05:50 +0530 Subject: [PATCH 05/27] changes Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 337 ++++++++++-------- 1 file changed, 185 insertions(+), 152 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index ff864f58171e..434aa3f8ad88 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -26,7 +26,9 @@ stopKubernetesCluster, deleteKubernetesCluster, upgradeKubernetesCluster, - scaleKubernetesCluster) + scaleKubernetesCluster, + destroyVirtualMachine, + deleteNetwork) from marvin.cloudstackException import CloudstackAPIException from marvin.codes import FAILED from marvin.lib.base import (Template, @@ -37,6 +39,7 @@ from marvin.lib.common import (get_zone) from marvin.sshClient import SshClient from nose.plugins.attrib import attr +from marvin.lib.decoratorGenerators import skipTestIf import time @@ -54,89 +57,89 @@ def setUpClass(cls): cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ cls.cks_template_name_key = "cloud.kubernetes.cluster.template.name." + cls.hypervisor.lower() + cls.hypervisorNotSupported = False + if cls.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + cls.hypervisorNotSupported = True cls.setup_failed = False - - cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, - name="cloud.kubernetes.service.enabled")[0].value - if cls.initial_configuration_cks_enabled not in ["true", True]: - cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") - Configurations.update(cls.apiclient, - "cloud.kubernetes.service.enabled", - "true") - cls.restartServer() - - cls.cks_template = None - cls.initial_configuration_cks_template_name = None - cls.cks_service_offering = None - + cls._cleanup = [] cls.kubernetes_version_ids = [] - if cls.setup_failed == False: - try: - cls.kubernetes_version_1 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.14.9"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_1.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.14.9"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.14.9"]["url"], e)) - if cls.setup_failed == False: - try: - cls.kubernetes_version_2 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.15.0"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_2.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.15.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.15.0"]["url"], e)) - if cls.setup_failed == False: - try: - cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.0"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.16.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.0"]["url"], e)) - if cls.setup_failed == False: - try: - cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.3"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) - - if cls.setup_failed == False: - cls.cks_template = Template.register( - cls.apiclient, - cls.services["cks_templates"][cls.hypervisor.lower()], - zoneid=cls.zone.id, - hypervisor=cls.hypervisor, - details=cks_template_data_details - ) - cls.debug("Waiting for CKS template with ID %s to be ready" % cls.cks_template.id) - try: - cls.waitForTemplateReadyState(cls.cks_template.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get CKS template in ready state, {}, {}".format(cks_template_data["url"], e)) - - cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, - name=cls.cks_template_name_key)[0].value - Configurations.update(cls.apiclient, - cls.cks_template_name_key, - cls.cks_template.name) - - cks_offering_data = cls.services["cks_service_offering"] - cks_offering_data["name"] = 'CKS-Instance-' + random_gen() - if cls.setup_failed == False: - cls.cks_service_offering = ServiceOffering.create( - cls.apiclient, - cks_offering_data - ) - cls._cleanup = [] - if cls.cks_template != None: - cls._cleanup.append(cls.cks_template) - if cls.cks_service_offering != None: - cls._cleanup.append(cls.cks_service_offering) + if cls.hypervisorNotSupported == False: + cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, + name="cloud.kubernetes.service.enabled")[0].value + if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") + Configurations.update(cls.apiclient, + "cloud.kubernetes.service.enabled", + "true") + cls.restartServer() + + cls.cks_template = None + cls.initial_configuration_cks_template_name = None + cls.cks_service_offering = None + + if cls.setup_failed == False: + try: + cls.kubernetes_version_1 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.14.9"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_1.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.14.9"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.14.9"]["url"], e)) + if cls.setup_failed == False: + try: + cls.kubernetes_version_2 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.15.0"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_2.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.15.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.15.0"]["url"], e)) + if cls.setup_failed == False: + try: + cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.0"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.16.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.0"]["url"], e)) + if cls.setup_failed == False: + try: + cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.3"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) + + if cls.setup_failed == False: + cls.cks_template = getKubernetesTemplate() + if template == FAILED: + assert False, "getKubernetesTemplate() failed to return template for hypervisor %s" % cls.hypervisor + cls.setup_failed = True + else: + cls._cleanup.append(cls.cks_template) + + if cls.setup_failed == False: + cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, + name=cls.cks_template_name_key)[0].value + Configurations.update(cls.apiclient, + cls.cks_template_name_key, + cls.cks_template.name) + + cks_offering_data = cls.services["cks_service_offering"] + cks_offering_data["name"] = 'CKS-Instance-' + random_gen() + cls.cks_service_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) + cls._cleanup.append(cls.cks_service_offering) + cls.domain = get_domain(cls.apiclient) + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) return @classmethod @@ -151,14 +154,10 @@ def tearDownClass(cls): cls.debug("Error: Exception during cleanup for added Kubernetes supported versions: %s" % e) try: # Restore original CKS template - if cls.initial_configuration_cks_template_name != None: + if clst.hypervisorNotSupported ==cls.initial_configuration_cks_template_name != None: Configurations.update(cls.apiclient, cls.cks_template_name_key, cls.initial_configuration_cks_template_name) - # Delete created CKS template - if cls.setup_failed == False and cls.cks_template != None: - cls.cks_template.delete(cls.apiclient, - cls.zone.id) # Restore CKS enabled if cls.initial_configuration_cks_enabled not in ["true", True]: cls.debug("Restoring Kubernetes Service enabled value") @@ -209,28 +208,39 @@ def isManagementUp(cls): return False @classmethod - def waitForTemplateReadyState(cls, template_id, retries=30, interval=60): - """Check if template download will finish""" - while retries > 0: - time.sleep(interval) - template_response = Template.list( - cls.apiclient, - id=template_id, - zoneid=cls.zone.id, - templatefilter='self' - ) + def getKubernetesTemplate(cls, cks_templates=None): - if isinstance(template_response, list): - template = template_response[0] - if not hasattr(template, 'status') or not template or not template.status: - retries = retries - 1 - continue - if 'Failed' == template.status: - raise Exception("Failed to download template: status - %s" % template.status) - elif template.status == 'Download Complete' and template.isready: - return - retries = retries - 1 - raise Exception("Template download timed out") + if cks_templates is None: + cks_templates = cls.services["cks_templates"] + + hypervisor = cls.hypervisor.lower() + + if hypervisor not in cks_templates.keys(): + print "Provided hypervisor has no CKS template" + return FAILED + + cks_template = cks_templates[hypervisor] + + cmd = listTemplates.listTemplatesCmd() + cmd.name = test_template['name'] + cmd.templatefilter = 'all' + cmd.zoneid = cls.zone.id + cmd.hypervisor = hypervisor + templates = apiclient.listTemplates(cmd) + + if validateList(templates)[0] != PASS: + details = None + if hypervisor not in ["vmware"] + details = cks_template["details"] + template = Template.register(apiclient, test_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False, details=detils) + template.download(apiclient) + return template + + for template in templates: + if template.isready and template.ispublic: + return template + + return FAILED @classmethod def waitForKubernetesSupportedVersionIsoReadyState(cls, version_id, retries=30, interval=60): @@ -282,6 +292,12 @@ def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() self.cleanup = [] + + # Delete any existing Kubernetes cluster for account + clusters = self.listKubernetesCluster() + for cluster in clusters: + self.deleteKubernetesClusterAndVerify(cluster.id, False, True) + return def tearDown(self): @@ -294,6 +310,7 @@ def tearDown(self): return @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") def test_01_deploy_kubernetes_cluster(self): """Test to deploy a new Kubernetes cluster @@ -302,10 +319,8 @@ def test_01_deploy_kubernetes_cluster(self): # 2. The Cloud Database contains the valid information # 3. stopKubernetesCluster should stop the cluster """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") + self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -319,13 +334,14 @@ def test_01_deploy_kubernetes_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully stopped, now deleting it" % cluster_response.id) - self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteKubernetesClusterAndVerify(cluster_response.id) self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") def test_02_deploy_kubernetes_ha_cluster(self): """Test to deploy a new Kubernetes cluster @@ -333,10 +349,8 @@ def test_02_deploy_kubernetes_ha_cluster(self): # 1. createKubernetesCluster should return valid info for new cluster # 2. The Cloud Database contains the valid information """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") + self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -346,13 +360,14 @@ def test_02_deploy_kubernetes_ha_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully deployed, now deleting it" % cluster_response.id) - self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteKubernetesClusterAndVerify(cluster_response.id) self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") def test_03_deploy_invalid_kubernetes_ha_cluster(self): """Test to deploy a new Kubernetes cluster @@ -360,16 +375,14 @@ def test_03_deploy_invalid_kubernetes_ha_cluster(self): # 1. createKubernetesCluster should return valid info for new cluster # 2. The Cloud Database contains the valid information """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") + self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) try: cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id, 1, 2) - self.debug("Invslid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % cluster_response.id) + self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % cluster_response.id) self.deleteKubernetesCluster(cluster_response.id) self.fail("HA Kubernetes cluster deployed with Kubernetes supported version below version 1.16.0. Must be an error.") except CloudstackAPIException as e: @@ -378,6 +391,7 @@ def test_03_deploy_invalid_kubernetes_ha_cluster(self): return @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") def test_04_deploy_and_upgrade_kubernetes_cluster(self): """Test to deploy a new Kubernetes cluster and upgrade it to newer version @@ -386,10 +400,8 @@ def test_04_deploy_and_upgrade_kubernetes_cluster(self): # 2. The Cloud Database contains the valid information # 3. upgradeKubernetesCluster should return valid info for the cluster """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") + self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -409,7 +421,7 @@ def test_04_deploy_and_upgrade_kubernetes_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) - self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteKubernetesClusterAndVerify(cluster_response.id) self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) @@ -417,6 +429,7 @@ def test_04_deploy_and_upgrade_kubernetes_cluster(self): @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): """Test to deploy a new HA Kubernetes cluster and upgrade it to newer version @@ -425,10 +438,8 @@ def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): # 2. The Cloud Database contains the valid information # 3. upgradeKubernetesCluster should return valid info for the cluster """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") + self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -448,13 +459,14 @@ def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) - self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteKubernetesClusterAndVerify(cluster_response.id) self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version @@ -463,10 +475,8 @@ def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): # 2. The Cloud Database contains the valid information # 3. upgradeKubernetesCluster should fail """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") + self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -486,13 +496,14 @@ def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): self.debug("Deleting Kubernetes cluster with ID: %s" % cluster_response.id) - self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteKubernetesClusterAndVerfiy(cluster_response.id, True) self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") def test_07_deploy_and_scale_kubernetes_cluster(self): """Test to deploy a new Kubernetes cluster and check for failure while tying to scale it @@ -502,10 +513,8 @@ def test_07_deploy_and_scale_kubernetes_cluster(self): # 3. scaleKubernetesCluster should return valid info for the cluster when it is scaled up # 4. scaleKubernetesCluster should return valid info for the cluster when it is scaled down """ - if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) if self.setup_failed == True: - self.skipTest("Setup incomplete") + self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -535,17 +544,20 @@ def test_07_deploy_and_scale_kubernetes_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully downscaled, now deleting it" % cluster_response.id) - self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteKubernetesClusterAndVerify(cluster_response.id) self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) return - def listKubernetesCluster(self, cluster_id): + def listKubernetesCluster(self, cluster_id = None): listKubernetesClustersCmd = listKubernetesClusters.listKubernetesClustersCmd() - listKubernetesClustersCmd.id = cluster_id + if cluster_id != None: + listKubernetesClustersCmd.id = cluster_id clusterResponse = self.apiclient.listKubernetesClusters(listKubernetesClustersCmd) - return clusterResponse[0] + if cluster_id != None: + return clusterResponse[0] + return clusterResponse def createKubernetesCluster(self, name, version_id, size=1, master_nodes=1): createKubernetesClusterCmd = createKubernetesCluster.createKubernetesClusterCmd() @@ -557,6 +569,8 @@ def createKubernetesCluster(self, name, version_id, size=1, master_nodes=1): createKubernetesClusterCmd.serviceofferingid = self.cks_service_offering.id createKubernetesClusterCmd.zoneid = self.zone.id createKubernetesClusterCmd.noderootdisksize = 10 + createKubernetesClusterCmd.account = self.account.name + createKubernetesClusterCmd.domainid = self.domain.id clusterResponse = self.apiclient.createKubernetesCluster(createKubernetesClusterCmd) if not clusterResponse: self.cleanup.append(clusterResponse) @@ -681,21 +695,40 @@ def stopAndVerifyKubernetesCluster(self, cluster_id): "KubernetesCluster not stopped in DB, {}".format(db_cluster_state) ) - def deleteAndVerifyKubernetesCluster(self, cluster_id): + def deleteKubernetesClusterAndVerify(self, cluster_id, verify = True, forced = False): """Delete Kubernetes cluster and check if it is really deleted""" - delete_response = self.deleteKubernetesCluster(cluster_id) - - self.assertEqual( - delete_response.success, - True, - "Check KubernetesCluster delete response {}, {}".format(delete_response.success, True) - ) + forceDeleted = False + try: + delete_response = self.deleteKubernetesCluster(cluster_id) + except Exception as e: + if forced: + cluster = self.listKubernetesCluster(cluster_id) + if cluster != None: + forceDeleted = True + for cluster_vm_id in cluster.virtualmachineids: + cmd = destroyVirtualMachine.destroyVirtualMachineCmd() + cmd.id = cluster_vm_id + cmd.expunge = True + self.apiclient.destroyVirtualMachine(cmd) + cmd = deleteNetwork.deleteNetworkCmd() + cmd.id = cluster.networkid + cmd.forced = True + self.apiclient.deleteNetwork(cmd) + + + + if verify == True and forceDeleted == False: + self.assertEqual( + delete_response.success, + True, + "Check KubernetesCluster delete response {}, {}".format(delete_response.success, True) + ) - db_cluster_removed = self.dbclient.execute("select removed from kubernetes_cluster where uuid = '%s';" % cluster_id)[0][0] + db_cluster_removed = self.dbclient.execute("select removed from kubernetes_cluster where uuid = '%s';" % cluster_id)[0][0] - self.assertNotEqual( - db_cluster_removed, - None, - "KubernetesCluster not removed in DB, {}".format(db_cluster_removed) - ) + self.assertNotEqual( + db_cluster_removed, + None, + "KubernetesCluster not removed in DB, {}".format(db_cluster_removed) + ) From 3a7776ddfd17dafe2ab39b49e845a53b3342234f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 28 Aug 2020 14:07:54 +0530 Subject: [PATCH 06/27] fix Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 434aa3f8ad88..69abac2da6c3 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -19,6 +19,7 @@ #Import Local Modules from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.cloudstackAPI import (listInfrastructure, + listTemplates, listKubernetesSupportedVersions, addKubernetesSupportedVersion, deleteKubernetesSupportedVersion, @@ -154,7 +155,7 @@ def tearDownClass(cls): cls.debug("Error: Exception during cleanup for added Kubernetes supported versions: %s" % e) try: # Restore original CKS template - if clst.hypervisorNotSupported ==cls.initial_configuration_cks_template_name != None: + if cls.hypervisorNotSupported == False and cls.initial_configuration_cks_template_name != None: Configurations.update(cls.apiclient, cls.cks_template_name_key, cls.initial_configuration_cks_template_name) @@ -216,7 +217,7 @@ def getKubernetesTemplate(cls, cks_templates=None): hypervisor = cls.hypervisor.lower() if hypervisor not in cks_templates.keys(): - print "Provided hypervisor has no CKS template" + cls.debug("Provided hypervisor has no CKS template") return FAILED cks_template = cks_templates[hypervisor] @@ -230,7 +231,7 @@ def getKubernetesTemplate(cls, cks_templates=None): if validateList(templates)[0] != PASS: details = None - if hypervisor not in ["vmware"] + if hypervisor not in ["vmware"]: details = cks_template["details"] template = Template.register(apiclient, test_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False, details=detils) template.download(apiclient) From 8f9d1bb5bfcd2439b6c1eb611e295e2e66827424 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 31 Aug 2020 13:44:58 +0530 Subject: [PATCH 07/27] fix Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 69abac2da6c3..ce0575fc44a1 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -113,7 +113,7 @@ def setUpClass(cls): (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) if cls.setup_failed == False: - cls.cks_template = getKubernetesTemplate() + cls.cks_template = cls.getKubernetesTemplate() if template == FAILED: assert False, "getKubernetesTemplate() failed to return template for hypervisor %s" % cls.hypervisor cls.setup_failed = True @@ -706,16 +706,20 @@ def deleteKubernetesClusterAndVerify(self, cluster_id, verify = True, forced = F if forced: cluster = self.listKubernetesCluster(cluster_id) if cluster != None: - forceDeleted = True - for cluster_vm_id in cluster.virtualmachineids: - cmd = destroyVirtualMachine.destroyVirtualMachineCmd() - cmd.id = cluster_vm_id - cmd.expunge = True - self.apiclient.destroyVirtualMachine(cmd) - cmd = deleteNetwork.deleteNetworkCmd() - cmd.id = cluster.networkid - cmd.forced = True - self.apiclient.deleteNetwork(cmd) + if cluster.state in ['Running', 'Upgrading', 'Scaling']: + self.stopKubernetesCluster(cluster_id) + self.deleteKubernetesCluster(cluster_id) + else: + forceDeleted = True + for cluster_vm_id in cluster.virtualmachineids: + cmd = destroyVirtualMachine.destroyVirtualMachineCmd() + cmd.id = cluster_vm_id + cmd.expunge = True + self.apiclient.destroyVirtualMachine(cmd) + cmd = deleteNetwork.deleteNetworkCmd() + cmd.id = cluster.networkid + cmd.forced = True + self.apiclient.deleteNetwork(cmd) From e03af5c9dd8f83be2f21fed13e982f51acd83270 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 31 Aug 2020 14:22:40 +0530 Subject: [PATCH 08/27] fix template var Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index ce0575fc44a1..c5dd525c4f7a 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -223,7 +223,7 @@ def getKubernetesTemplate(cls, cks_templates=None): cks_template = cks_templates[hypervisor] cmd = listTemplates.listTemplatesCmd() - cmd.name = test_template['name'] + cmd.name = cks_template['name'] cmd.templatefilter = 'all' cmd.zoneid = cls.zone.id cmd.hypervisor = hypervisor @@ -233,7 +233,7 @@ def getKubernetesTemplate(cls, cks_templates=None): details = None if hypervisor not in ["vmware"]: details = cks_template["details"] - template = Template.register(apiclient, test_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False, details=detils) + template = Template.register(apiclient, cks_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False, details=detils) template.download(apiclient) return template From 2a167b0509e9aa11f35b22fffa429dd9336f9686 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 31 Aug 2020 16:38:02 +0530 Subject: [PATCH 09/27] fix Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index c5dd525c4f7a..af89591f33e1 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -227,14 +227,14 @@ def getKubernetesTemplate(cls, cks_templates=None): cmd.templatefilter = 'all' cmd.zoneid = cls.zone.id cmd.hypervisor = hypervisor - templates = apiclient.listTemplates(cmd) + templates = cls.apiclient.listTemplates(cmd) if validateList(templates)[0] != PASS: details = None if hypervisor not in ["vmware"]: details = cks_template["details"] - template = Template.register(apiclient, cks_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False, details=detils) - template.download(apiclient) + template = Template.register(cls.apiclient, cks_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False, details=details) + template.download(cls.apiclient) return template for template in templates: From 639dac3bb2ecdac99a60c47d162fdedb99e7068b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 31 Aug 2020 18:45:02 +0530 Subject: [PATCH 10/27] fix Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index af89591f33e1..ef85f822cc9a 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -23,6 +23,7 @@ listKubernetesSupportedVersions, addKubernetesSupportedVersion, deleteKubernetesSupportedVersion, + listKubernetesClusters, createKubernetesCluster, stopKubernetesCluster, deleteKubernetesCluster, @@ -31,13 +32,16 @@ destroyVirtualMachine, deleteNetwork) from marvin.cloudstackException import CloudstackAPIException -from marvin.codes import FAILED +from marvin.codes import PASS, FAILED from marvin.lib.base import (Template, ServiceOffering, + Account, Configurations) from marvin.lib.utils import (cleanup_resources, + validateList, random_gen) -from marvin.lib.common import (get_zone) +from marvin.lib.common import (get_zone, + get_domain) from marvin.sshClient import SshClient from nose.plugins.attrib import attr from marvin.lib.decoratorGenerators import skipTestIf @@ -95,26 +99,26 @@ def setUpClass(cls): cls.setup_failed = True cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % (cls.services["cks_kubernetes_versions"]["1.15.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.15.0"]["url"], e)) - if cls.setup_failed == False: - try: - cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.0"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.16.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.0"]["url"], e)) - if cls.setup_failed == False: - try: - cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.3"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) - except Exception as e: - cls.setup_failed = True - cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) +# if cls.setup_failed == False: +# try: +# cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.0"]) +# cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) +# except Exception as e: +# cls.setup_failed = True +# cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % +# (cls.services["cks_kubernetes_versions"]["1.16.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.0"]["url"], e)) +# if cls.setup_failed == False: +# try: +# cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.3"]) +# cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) +# except Exception as e: +# cls.setup_failed = True +# cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % +# (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) if cls.setup_failed == False: cls.cks_template = cls.getKubernetesTemplate() - if template == FAILED: + if cls.cks_template == FAILED: assert False, "getKubernetesTemplate() failed to return template for hypervisor %s" % cls.hypervisor cls.setup_failed = True else: @@ -231,9 +235,9 @@ def getKubernetesTemplate(cls, cks_templates=None): if validateList(templates)[0] != PASS: details = None - if hypervisor not in ["vmware"]: + if hypervisor in ["vmware"] and "details" in cks_template: details = cks_template["details"] - template = Template.register(cls.apiclient, cks_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False, details=details) + template = Template.register(cls.apiclient, cks_template, zoneid=cls.zone.id, hypervisor=hypervisor.lower(), randomize_name=False, details=details) template.download(cls.apiclient) return template @@ -296,8 +300,9 @@ def setUp(self): # Delete any existing Kubernetes cluster for account clusters = self.listKubernetesCluster() - for cluster in clusters: - self.deleteKubernetesClusterAndVerify(cluster.id, False, True) + if clusters != None: + for cluster in clusters: + self.deleteKubernetesClusterAndVerify(cluster.id, False, True) return From adc63332e8b1958e93bae70eb0fade0be7c63980 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 1 Sep 2020 00:20:09 +0530 Subject: [PATCH 11/27] fix cleanup Signed-off-by: Abhishek Kumar --- .../integration/smoke/test_kubernetes_clusters.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index ef85f822cc9a..3c2033a274ec 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -243,7 +243,7 @@ def getKubernetesTemplate(cls, cks_templates=None): for template in templates: if template.isready and template.ispublic: - return template + return Template(template.__dict__) return FAILED @@ -297,18 +297,15 @@ def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() self.cleanup = [] - - # Delete any existing Kubernetes cluster for account - clusters = self.listKubernetesCluster() - if clusters != None: - for cluster in clusters: - self.deleteKubernetesClusterAndVerify(cluster.id, False, True) - return def tearDown(self): try: - #Clean up, terminate the created templates + # Delete any existing Kubernetes cluster for account + clusters = self.listKubernetesCluster() + if clusters != None: + for cluster in clusters: + self.deleteKubernetesClusterAndVerify(cluster.id, False, True) cleanup_resources(self.apiclient, self.cleanup) except Exception as e: From 3d5a83e96d0462a1fd484a6b89f63bcf926e4611 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 1 Sep 2020 00:26:58 +0530 Subject: [PATCH 12/27] uncomment versions Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 3c2033a274ec..9a30a2e47f8d 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -99,22 +99,22 @@ def setUpClass(cls): cls.setup_failed = True cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % (cls.services["cks_kubernetes_versions"]["1.15.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.15.0"]["url"], e)) -# if cls.setup_failed == False: -# try: -# cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.0"]) -# cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) -# except Exception as e: -# cls.setup_failed = True -# cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % -# (cls.services["cks_kubernetes_versions"]["1.16.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.0"]["url"], e)) -# if cls.setup_failed == False: -# try: -# cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.3"]) -# cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) -# except Exception as e: -# cls.setup_failed = True -# cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % -# (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) + if cls.setup_failed == False: + try: + cls.kubernetes_version_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.0"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_3.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.16.0"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.0"]["url"], e)) + if cls.setup_failed == False: + try: + cls.kubernetes_version_4 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.16.3"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_4.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % + (cls.services["cks_kubernetes_versions"]["1.16.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.16.3"]["url"], e)) if cls.setup_failed == False: cls.cks_template = cls.getKubernetesTemplate() From d085b85401bf64d468d04615b410065143d7f2f7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 2 Sep 2020 11:57:50 +0530 Subject: [PATCH 13/27] refactor error messages, api param description Signed-off-by: Abhishek Kumar --- .../kubernetes/version/KubernetesVersionManagerImpl.java | 8 ++++---- .../version/AddKubernetesSupportedVersionCmd.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 5f2e42b11d52..229e912d1cee 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -181,10 +181,10 @@ public static int compareSemanticVersions(String v1, String v2) throws IllegalAr throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", v1, v2)); } if(!isSemanticVersion(v1)) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", v1)); + throw new IllegalArgumentException(String.format("Invalid version format, %s. Semantic version should be specified in MAJOR.MINOR.PATCH format", v1)); } if(!isSemanticVersion(v2)) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", v2)); + throw new IllegalArgumentException(String.format("Invalid version format, %s. Semantic version should be specified in MAJOR.MINOR.PATCH format", v2)); } String[] thisParts = v1.split("\\."); String[] thatParts = v2.split("\\."); @@ -287,10 +287,10 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad final Integer minimumCpu = cmd.getMinimumCpu(); final Integer minimumRamSize = cmd.getMinimumRamSize(); if (minimumCpu == null || minimumCpu < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU) { - throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.MIN_CPU_NUMBER)); + throw new InvalidParameterValueException(String.format("Invalid value for %s parameter. Minimum %d vCPUs required.", ApiConstants.MIN_CPU_NUMBER, KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU)); } if (minimumRamSize == null || minimumRamSize < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { - throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.MIN_MEMORY)); + throw new InvalidParameterValueException(String.format("Invalid value for %s parameter. Minimum %dMB memory required", KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); } if (compareSemanticVersions(semanticVersion, MIN_KUBERNETES_VERSION) < 0) { throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index a85e6ee064ac..ece9d5aae5d8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -61,7 +61,7 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCm private String name; @Parameter(name = ApiConstants.SEMANTIC_VERSION, type = CommandType.STRING, required = true, - description = "the semantic version of the Kubernetes version") + description = "the semantic version of the Kubernetes version. It needs to be specified in MAJOR.MINOR.PATCH format") private String semanticVersion; @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, From 26c23f9331702010681f827753701bfb63f11fca Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 2 Sep 2020 12:53:51 +0530 Subject: [PATCH 14/27] fix Signed-off-by: Abhishek Kumar --- .../cloud/kubernetes/version/KubernetesVersionManagerImpl.java | 2 +- test/integration/smoke/test_kubernetes_clusters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 229e912d1cee..72a1c3794871 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -290,7 +290,7 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad throw new InvalidParameterValueException(String.format("Invalid value for %s parameter. Minimum %d vCPUs required.", ApiConstants.MIN_CPU_NUMBER, KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU)); } if (minimumRamSize == null || minimumRamSize < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { - throw new InvalidParameterValueException(String.format("Invalid value for %s parameter. Minimum %dMB memory required", KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); + throw new InvalidParameterValueException(String.format("Invalid value for %s parameter. Minimum %dMB memory required", ApiConstants.MIN_MEMORY, KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); } if (compareSemanticVersions(semanticVersion, MIN_KUBERNETES_VERSION) < 0) { throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 9a30a2e47f8d..7a7bcc2203fc 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -237,7 +237,7 @@ def getKubernetesTemplate(cls, cks_templates=None): details = None if hypervisor in ["vmware"] and "details" in cks_template: details = cks_template["details"] - template = Template.register(cls.apiclient, cks_template, zoneid=cls.zone.id, hypervisor=hypervisor.lower(), randomize_name=False, details=details) + template = Template.register(cls.apiclient, cks_template, zoneid=None, hypervisor=hypervisor.lower(), randomize_name=False, details=details) template.download(cls.apiclient) return template From 395096334c7eeefb35a0e120f15542253da21f64 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 7 Sep 2020 12:00:20 +0530 Subject: [PATCH 15/27] fix deleted template state Signed-off-by: Abhishek Kumar --- .../com/cloud/storage/dao/VMTemplateDaoImpl.java | 8 +++++++- .../cluster/KubernetesClusterManagerImpl.java | 16 +++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 1621baef2dfd..dabeb99832ae 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -95,6 +95,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem protected SearchBuilder AccountIdSearch; protected SearchBuilder NameSearch; + protected SearchBuilder ValidNameSearch; protected SearchBuilder TmpltsInZoneSearch; protected SearchBuilder ActiveTmpltSearch; private SearchBuilder PublicSearch; @@ -138,8 +139,9 @@ public VMTemplateVO findByTemplateName(String templateName) { @Override public VMTemplateVO findValidByTemplateName(String templateName) { - SearchCriteria sc = NameSearch.create(); + SearchCriteria sc = ValidNameSearch.create(); sc.setParameters("name", templateName); + sc.setParameters("state", VirtualMachineTemplate.State.Active); return findOneBy(sc); } @@ -319,6 +321,10 @@ public boolean configure(String name, Map params) throws Configu UniqueNameSearch.and("uniqueName", UniqueNameSearch.entity().getUniqueName(), SearchCriteria.Op.EQ); NameSearch = createSearchBuilder(); NameSearch.and("name", NameSearch.entity().getName(), SearchCriteria.Op.EQ); + ValidNameSearch = createSearchBuilder(); + ValidNameSearch.and("name", ValidNameSearch.entity().getName(), SearchCriteria.Op.EQ); + ValidNameSearch.and("state", ValidNameSearch.entity().getName(), SearchCriteria.Op.EQ); + ValidNameSearch.and("removed", ValidNameSearch.entity().getName(), SearchCriteria.Op.NULL); NameAccountIdSearch = createSearchBuilder(); NameAccountIdSearch.and("name", NameAccountIdSearch.entity().getName(), SearchCriteria.Op.EQ); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 204f2d7b83da..00c7ef1f925b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.kubernetes.cluster; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; @@ -292,7 +294,7 @@ private boolean isKubernetesServiceTemplateConfigured(DataCenter zone) { LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", templateKey)); return false; } - final VMTemplateVO template = templateDao.findByTemplateName(templateName); + final VMTemplateVO template = templateDao.findValidByTemplateName(templateName); if (template == null) { LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster nodes", templateName)); return false; @@ -375,22 +377,22 @@ private IpAddress getSourceNatIp(Network network) { } private VMTemplateVO getKubernetesServiceTemplate(Hypervisor.HypervisorType hypervisorType) { - String tempalteName = null; + String templateName = null; switch (hypervisorType) { case Hyperv: - tempalteName = KubernetesClusterHyperVTemplateName.value(); + templateName = KubernetesClusterHyperVTemplateName.value(); break; case KVM: - tempalteName = KubernetesClusterKVMTemplateName.value(); + templateName = KubernetesClusterKVMTemplateName.value(); break; case VMware: - tempalteName = KubernetesClusterVMwareTemplateName.value(); + templateName = KubernetesClusterVMwareTemplateName.value(); break; case XenServer: - tempalteName = KubernetesClusterXenserverTemplateName.value(); + templateName = KubernetesClusterXenserverTemplateName.value(); break; } - return templateDao.findValidByTemplateName(tempalteName); + return templateDao.findValidByTemplateName(templateName); } private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCount) { From 741684d6849d7e9084edfc15387aa18944007caf Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 7 Sep 2020 12:12:25 +0530 Subject: [PATCH 16/27] fix Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 7a7bcc2203fc..4543a2626370 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -159,6 +159,8 @@ def tearDownClass(cls): cls.debug("Error: Exception during cleanup for added Kubernetes supported versions: %s" % e) try: # Restore original CKS template + if cls.cks_template != None: + cls.cks_template.delete(cls.apiclient) if cls.hypervisorNotSupported == False and cls.initial_configuration_cks_template_name != None: Configurations.update(cls.apiclient, cls.cks_template_name_key, @@ -237,7 +239,7 @@ def getKubernetesTemplate(cls, cks_templates=None): details = None if hypervisor in ["vmware"] and "details" in cks_template: details = cks_template["details"] - template = Template.register(cls.apiclient, cks_template, zoneid=None, hypervisor=hypervisor.lower(), randomize_name=False, details=details) + template = Template.register(cls.apiclient, cks_template, zoneid=cls.zone.id, hypervisor=hypervisor.lower(), randomize_name=False, details=details) template.download(cls.apiclient) return template From a1b86216c49734cc65ebfe7362e6fc3e574e07a2 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 7 Sep 2020 13:29:35 +0530 Subject: [PATCH 17/27] fix Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index dabeb99832ae..6caef20b2d7a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -323,8 +323,8 @@ public boolean configure(String name, Map params) throws Configu NameSearch.and("name", NameSearch.entity().getName(), SearchCriteria.Op.EQ); ValidNameSearch = createSearchBuilder(); ValidNameSearch.and("name", ValidNameSearch.entity().getName(), SearchCriteria.Op.EQ); - ValidNameSearch.and("state", ValidNameSearch.entity().getName(), SearchCriteria.Op.EQ); - ValidNameSearch.and("removed", ValidNameSearch.entity().getName(), SearchCriteria.Op.NULL); + ValidNameSearch.and("state", ValidNameSearch.entity().getState(), SearchCriteria.Op.EQ); + ValidNameSearch.and("removed", ValidNameSearch.entity().getRemoved(), SearchCriteria.Op.NULL); NameAccountIdSearch = createSearchBuilder(); NameAccountIdSearch.and("name", NameAccountIdSearch.entity().getName(), SearchCriteria.Op.EQ); From 4e8dd1f5aa7b3030dd894dd70cdd3c6c7ee7c51e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 7 Sep 2020 17:27:21 +0530 Subject: [PATCH 18/27] fix deleted network check Signed-off-by: Abhishek Kumar --- .../KubernetesClusterDestroyWorker.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index c1a095fedbe9..41de39fddc2a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -198,24 +198,25 @@ public boolean destroy() throws CloudRuntimeException { } if (cleanupNetwork) { // if network has additional VM, cannot proceed with cluster destroy NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network == null) { - logAndThrow(Level.ERROR, String.format("Failed to find network for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - List networkVMs = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(network.getId(), VirtualMachine.Type.User); - if (networkVMs.size() > clusterVMs.size()) { - logAndThrow(Level.ERROR, String.format("Network ID: %s for Kubernetes cluster ID: %s has instances using it which are not part of the Kubernetes cluster", network.getUuid(), kubernetesCluster.getUuid())); - } - for (VMInstanceVO vm : networkVMs) { - boolean vmFoundInKubernetesCluster = false; - for (KubernetesClusterVmMap clusterVM : clusterVMs) { - if (vm.getId() == clusterVM.getVmId()) { - vmFoundInKubernetesCluster = true; - break; - } + if (network != null) { + List networkVMs = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(network.getId(), VirtualMachine.Type.User); + if (networkVMs.size() > clusterVMs.size()) { + logAndThrow(Level.ERROR, String.format("Network ID: %s for Kubernetes cluster ID: %s has instances using it which are not part of the Kubernetes cluster", network.getUuid(), kubernetesCluster.getUuid())); } - if (!vmFoundInKubernetesCluster) { - logAndThrow(Level.ERROR, String.format("VM ID: %s which is not a part of Kubernetes cluster ID: %s is using Kubernetes cluster network ID: %s", vm.getUuid(), kubernetesCluster.getUuid(), network.getUuid())); + for (VMInstanceVO vm : networkVMs) { + boolean vmFoundInKubernetesCluster = false; + for (KubernetesClusterVmMap clusterVM : clusterVMs) { + if (vm.getId() == clusterVM.getVmId()) { + vmFoundInKubernetesCluster = true; + break; + } + } + if (!vmFoundInKubernetesCluster) { + logAndThrow(Level.ERROR, String.format("VM ID: %s which is not a part of Kubernetes cluster ID: %s is using Kubernetes cluster network ID: %s", vm.getUuid(), kubernetesCluster.getUuid(), network.getUuid())); + } } + } else { + LOGGER.error(String.format("Failed to find network for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } } if (LOGGER.isInfoEnabled()) { From e86914bcf4c5a77cd032599a4bd90109d5c2a2b5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 8 Sep 2020 12:02:07 +0530 Subject: [PATCH 19/27] fix capacity check Signed-off-by: Abhishek Kumar --- .../cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java | 2 +- .../KubernetesClusterResourceModifierActionWorker.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 00c7ef1f925b..8983e0aa0473 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -516,7 +516,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin } boolean suitable_host_found = false; Cluster planCluster = null; - for (int i = 1; i <= nodesCount + 1; i++) { + for (int i = 1; i <= nodesCount; i++) { suitable_host_found = false; for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { Pair hp = hostEntry.getValue(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 8b6df146bff0..831bebe5ff2d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -205,7 +205,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); } boolean suitable_host_found = false; - for (int i = 1; i <= nodesCount + 1; i++) { + for (int i = 1; i <= nodesCount; i++) { suitable_host_found = false; for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { Pair hp = hostEntry.getValue(); From 3b594a7260c12f1ab0d864b77a377bac036a5c0c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 8 Sep 2020 12:54:21 +0530 Subject: [PATCH 20/27] logging fix Signed-off-by: Abhishek Kumar --- .../kubernetes/cluster/KubernetesClusterManagerImpl.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 8983e0aa0473..a6538c946c34 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -16,8 +16,6 @@ // under the License. package com.cloud.kubernetes.cluster; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; @@ -993,7 +991,7 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) try { deployDestination = plan(totalNodeCount, zone, serviceOffering); } catch (InsufficientCapacityException e) { - logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d cluster nodes in zone ID: %s with service offering ID: %s", totalNodeCount, zone.getUuid(), serviceOffering.getUuid())); + logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d nodes cluster in zone ID: %s with service offering ID: %s", totalNodeCount, zone.getUuid(), serviceOffering.getUuid())); } if (deployDestination == null || deployDestination.getCluster() == null) { logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone ID: %s", zone.getUuid())); From b36a800d4b4affe8eb9cc886f238ca4b55c1225d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 8 Sep 2020 20:13:04 +0530 Subject: [PATCH 21/27] fix typo Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 4543a2626370..e10cd5ac1783 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -501,7 +501,7 @@ def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): self.debug("Deleting Kubernetes cluster with ID: %s" % cluster_response.id) - self.deleteKubernetesClusterAndVerfiy(cluster_response.id, True) + self.deleteKubernetesClusterAndVerify(cluster_response.id, True) self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) From ce5ebe9c1cbf424930814399c73dbc5bf0b8d51c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 15 Sep 2020 15:50:54 +0530 Subject: [PATCH 22/27] refactor tests Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 311 ++++++++++-------- 1 file changed, 166 insertions(+), 145 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index e10cd5ac1783..f2ec874fa20b 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -26,6 +26,7 @@ listKubernetesClusters, createKubernetesCluster, stopKubernetesCluster, + startKubernetesCluster, deleteKubernetesCluster, upgradeKubernetesCluster, scaleKubernetesCluster, @@ -50,6 +51,8 @@ _multiprocess_shared_ = True +k8s_cluster = None + class TestKubernetesCluster(cloudstackTestCase): @classmethod @@ -176,7 +179,7 @@ def tearDownClass(cls): cleanup_resources(cls.apiclient, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) - if version_delete_failed == True: + if version_delete_failed == True: raise Exception("Warning: Exception during cleanup, unable to delete Kubernetes supported versions") return @@ -273,7 +276,7 @@ def listKubernetesSupportedVersion(cls, version_id): versionResponse = cls.apiclient.listKubernetesSupportedVersions(listKubernetesSupportedVersionsCmd) return versionResponse[0] - @classmethod + @classmethod def addKubernetesSupportedVersion(cls, version_service): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() addKubernetesSupportedVersionCmd.semanticversion = version_service["semanticversion"] @@ -303,13 +306,7 @@ def setUp(self): def tearDown(self): try: - # Delete any existing Kubernetes cluster for account - clusters = self.listKubernetesCluster() - if clusters != None: - for cluster in clusters: - self.deleteKubernetesClusterAndVerify(cluster.id, False, True) cleanup_resources(self.apiclient, self.cleanup) - except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -326,232 +323,220 @@ def test_01_deploy_kubernetes_cluster(self): """ if self.setup_failed == True: self.fail("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) - - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id) + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_2.id) + self.debug("Kubernetes cluster with ID: %s successfully deployed, now stopping it" % k8s_cluster.id) - self.debug("Kubernetes cluster with ID: %s successfully deployed, now stopping it" % cluster_response.id) + self.stopAndVerifyKubernetesCluster(k8s_cluster.id) - self.stopAndVerifyKubernetesCluster(cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully stopped, now starting it again" % k8s_cluster.id) - self.debug("Kubernetes cluster with ID: %s successfully stopped, now deleting it" % cluster_response.id) - - self.deleteKubernetesClusterAndVerify(cluster_response.id) + try: + k8s_cluster = self.startKubernetesCluster(k8s_cluster.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to start Kubernetes cluster due to: %s" % e) - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.verifyKubernetesClusterState(k8s_cluster, 'Running') return @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") - def test_02_deploy_kubernetes_ha_cluster(self): - """Test to deploy a new Kubernetes cluster + def test_02_invalid_upgrade_kubernetes_cluster(self): + """Test to check for failure while tying to upgrade a Kubernetes cluster to a lower version # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information + # 1. upgradeKubernetesCluster should fail """ if self.setup_failed == True: self.fail("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) - - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_3.id, 1, 2) + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_3.id, 1, 2) + self.debug("Upgrading Kubernetes cluster with ID: %s to a lower version" % k8s_cluster.id) - self.debug("Kubernetes cluster with ID: %s successfully deployed, now deleting it" % cluster_response.id) - - self.deleteKubernetesClusterAndVerify(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1.id) + self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kubernetes_version_1.id) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") + except Exception as e: + self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) return @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") - def test_03_deploy_invalid_kubernetes_ha_cluster(self): - """Test to deploy a new Kubernetes cluster + def test_03_deploy_and_upgrade_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and upgrade it to newer version # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information + # 1. upgradeKubernetesCluster should return valid info for the cluster """ if self.setup_failed == True: self.fail("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() + + self.debug("Upgrading Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id, 1, 2) - self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % cluster_response.id) - self.deleteKubernetesCluster(cluster_response.id) - self.fail("HA Kubernetes cluster deployed with Kubernetes supported version below version 1.16.0. Must be an error.") - except CloudstackAPIException as e: - self.debug("HA Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_3.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_3.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") - def test_04_deploy_and_upgrade_kubernetes_cluster(self): - """Test to deploy a new Kubernetes cluster and upgrade it to newer version + def test_04_deploy_and_scale_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and check for failure while tying to scale it # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information - # 3. upgradeKubernetesCluster should return valid info for the cluster + # 1. scaleKubernetesCluster should return valid info for the cluster when it is scaled up + # 2. scaleKubernetesCluster should return valid info for the cluster when it is scaled down """ if self.setup_failed == True: self.fail("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) - - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id) - - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_2.id) + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() - self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) + self.debug("Upscaling Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kubernetes_version_3.id) + k8s_cluster = self.scaleKubernetesCluster(k8s_cluster.id, 2) except Exception as e: - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Failed to upgrade Kubernetes cluster due to: %s" % e) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upscale Kubernetes cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(cluster_response, self.kubernetes_version_3.id) + self.verifyKubernetesClusterScale(k8s_cluster, 2) - self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully upscaled, now downscaling it" % k8s_cluster.id) + + try: + k8s_cluster = self.scaleKubernetesCluster(k8s_cluster.id, 1) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to downscale Kubernetes cluster due to: %s" % e) - self.deleteKubernetesClusterAndVerify(cluster_response.id) + self.verifyKubernetesClusterScale(k8s_cluster) - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully downscaled" % k8s_cluster.id) return - @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") - def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): - """Test to deploy a new HA Kubernetes cluster and upgrade it to newer version + def test_05_delete_kubernetes_cluster(self): + """Test to delete an existing Kubernetes cluster # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information - # 3. upgradeKubernetesCluster should return valid info for the cluster + # 1. deleteKubernetesCluster should delete an existing Kubernetes cluster """ if self.setup_failed == True: self.fail("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster() - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_3.id, 1, 2) + self.debug("Deleting Kubernetes cluster with ID: %s" % k8s_cluster.id) - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_3.id, 1, 2) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id) - self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deleted" % k8s_cluster.id) - try: - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kubernetes_version_4.id) - except Exception as e: - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(cluster_response, self.kubernetes_version_4.id) - - self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) - - self.deleteKubernetesClusterAndVerify(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + k8s_cluster = None return @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") - def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): - """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version + def test_06_deploy_invalid_kubernetes_ha_cluster(self): + """Test to deploy an invalid HA Kubernetes cluster # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information - # 3. upgradeKubernetesCluster should fail + # 1. createKubernetesCluster should fail as version doesn't support HA """ if self.setup_failed == True: self.fail("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id) - - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_2.id) - - self.debug("Kubernetes cluster with ID: %s successfully deployed, now scaling it" % cluster_response.id) - try: - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kubernetes_version_1.id) - self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kubernetes_version_1.id) - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") - except Exception as e: - self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) - - self.debug("Deleting Kubernetes cluster with ID: %s" % cluster_response.id) - - self.deleteKubernetesClusterAndVerify(cluster_response.id, True) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id, 1, 2) + self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % cluster_response.id) + self.deleteKubernetesClusterAndVerify(cluster_response.id, False, True) + self.fail("HA Kubernetes cluster deployed with Kubernetes supported version below version 1.16.0. Must be an error.") + except CloudstackAPIException as e: + self.debug("HA Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) return @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") - def test_07_deploy_and_scale_kubernetes_cluster(self): - """Test to deploy a new Kubernetes cluster and check for failure while tying to scale it + def test_07_deploy_kubernetes_ha_cluster(self): + """Test to deploy a new Kubernetes cluster # Validate the following: # 1. createKubernetesCluster should return valid info for new cluster # 2. The Cloud Database contains the valid information - # 3. scaleKubernetesCluster should return valid info for the cluster when it is scaled up - # 4. scaleKubernetesCluster should return valid info for the cluster when it is scaled down """ if self.setup_failed == True: self.fail("Setup incomplete") - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 2) - cluster_response = self.createKubernetesCluster(name, self.kubernetes_version_2.id) + self.debug("HA Kubernetes cluster with ID: %s successfully deployed" % k8s_cluster.id) - self.verifyKubernetesCluster(cluster_response, name, self.kubernetes_version_2.id) + return - self.debug("Kubernetes cluster with ID: %s successfully deployed, now upscaling it" % cluster_response.id) + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_deploy_and_upgrade_kubernetes_ha_cluster(self): + """Test to deploy a new HA Kubernetes cluster and upgrade it to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 2) try: - cluster_response = self.scaleKubernetesCluster(cluster_response.id, 2) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_4.id) except Exception as e: - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Failed to upscale Kubernetes cluster due to: %s" % e) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - self.verifyKubernetesClusterScale(cluster_response, 2) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_4.id) - self.debug("Kubernetes cluster with ID: %s successfully upscaled, now downscaling it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - try: - cluster_response = self.scaleKubernetesCluster(cluster_response.id, 1) - except Exception as e: - self.deleteKubernetesCluster(cluster_response.id) - self.fail("Failed to downscale Kubernetes cluster due to: %s" % e) + return - self.verifyKubernetesClusterScale(cluster_response) + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_09_delete_kubernetes_ha_cluster(self): + """Test to delete a HA Kubernetes cluster - self.debug("Kubernetes cluster with ID: %s successfully downscaled, now deleting it" % cluster_response.id) + # Validate the following: + # 1. deleteKubernetesCluster should delete an existing HA Kubernetes cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 2) + + self.debug("Deleting Kubernetes cluster with ID: %s" % k8s_cluster.id) - self.deleteKubernetesClusterAndVerify(cluster_response.id) + self.deleteKubernetesClusterAndVerify(k8s_cluster.id) - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deleted" % k8s_cluster.id) return @@ -560,7 +545,7 @@ def listKubernetesCluster(self, cluster_id = None): if cluster_id != None: listKubernetesClustersCmd.id = cluster_id clusterResponse = self.apiclient.listKubernetesClusters(listKubernetesClustersCmd) - if cluster_id != None: + if cluster_id != None and clusterResponse != None: return clusterResponse[0] return clusterResponse @@ -587,6 +572,12 @@ def stopKubernetesCluster(self, cluster_id): response = self.apiclient.stopKubernetesCluster(stopKubernetesClusterCmd) return response + def startKubernetesCluster(self, cluster_id): + startKubernetesClusterCmd = startKubernetesCluster.startKubernetesClusterCmd() + startKubernetesClusterCmd.id = cluster_id + response = self.apiclient.startnetesCluster(startKubernetesClusterCmd) + return response + def deleteKubernetesCluster(self, cluster_id): deleteKubernetesClusterCmd = deleteKubernetesCluster.deleteKubernetesClusterCmd() deleteKubernetesClusterCmd.id = cluster_id @@ -607,18 +598,47 @@ def scaleKubernetesCluster(self, cluster_id, size): response = self.apiclient.scaleKubernetesCluster(scaleKubernetesClusterCmd) return response - def verifyKubernetesCluster(self, cluster_response, name, version_id, size=1, master_nodes=1): + def getValidKubernetesCluster(self, size=1, master_nodes=1): + cluster = k8s_cluster + version = self.kubernetes_version_2 + if master_nodes != 1: + version = self.kubernetes_version_3 + valid = True + if cluster == None: + valid = False + self.debug("No existing cluster available, k8s_cluster: %s" % cluster) + if valid == True and cluster.id == None: + valid = False + self.debug("ID for existing cluster not found, k8s_cluster ID: %s" % cluster.id) + if valid == True: + try: + self.verifyKubernetesCluster(cluster, cluster.name, None, size, master_nodes) + self.debug("Existing Kubernetes cluster available with name %s" % cluster.name) + except AssertionError as error: + valid = False + self.debug("Existing cluster failed verification due to %s, need to deploy a new one" % error) + self.deleteKubernetesClusterAndVerify(cluster.id, False, True) + if valid == False: + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) + cluster = self.createKubernetesCluster(name, version.id, size, master_nodes) + self.verifyKubernetesCluster(cluster, name, version.id, size, master_nodes) + return cluster + + def verifyKubernetesCluster(self, cluster_response, name, version_id=None, size=1, master_nodes=1): """Check if Kubernetes cluster is valid""" self.verifyKubernetesClusterState(cluster_response, 'Running') - self.assertEqual( - cluster_response.name, - name, - "Check KubernetesCluster name {}, {}".format(cluster_response.name, name) - ) + if name != None: + self.assertEqual( + cluster_response.name, + name, + "Check KubernetesCluster name {}, {}".format(cluster_response.name, name) + ) - self.verifyKubernetesClusterVersion(cluster_response, version_id) + if version_id != None: + self.verifyKubernetesClusterVersion(cluster_response, version_id) self.assertEqual( cluster_response.zoneid, @@ -710,7 +730,7 @@ def deleteKubernetesClusterAndVerify(self, cluster_id, verify = True, forced = F if forced: cluster = self.listKubernetesCluster(cluster_id) if cluster != None: - if cluster.state in ['Running', 'Upgrading', 'Scaling']: + if cluster.state in ['Starting', 'Running', 'Upgrading', 'Scaling']: self.stopKubernetesCluster(cluster_id) self.deleteKubernetesCluster(cluster_id) else: @@ -724,8 +744,9 @@ def deleteKubernetesClusterAndVerify(self, cluster_id, verify = True, forced = F cmd.id = cluster.networkid cmd.forced = True self.apiclient.deleteNetwork(cmd) - - + self.dbclient.execute("update kubernetes_cluster set state='Destroyed', removed=now() where uuid = '%s';" % cluster.id) + else: + raise Exception("Error: Exception during delete cluster : %s" % e) if verify == True and forceDeleted == False: self.assertEqual( From 41351655827b9eb34f75651e7331c8336c9053d9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 15 Sep 2020 18:23:10 +0530 Subject: [PATCH 23/27] fix event description Signed-off-by: Abhishek Kumar --- .../version/DeleteKubernetesSupportedVersionCmd.java | 9 ++++++++- .../kubernetes/cluster/CreateKubernetesClusterCmd.java | 2 +- .../kubernetes/cluster/DeleteKubernetesClusterCmd.java | 8 +++++++- .../kubernetes/cluster/ScaleKubernetesClusterCmd.java | 8 +++++++- .../kubernetes/cluster/StartKubernetesClusterCmd.java | 8 +++++++- .../kubernetes/cluster/StopKubernetesClusterCmd.java | 8 +++++++- .../kubernetes/cluster/UpgradeKubernetesClusterCmd.java | 8 +++++++- 7 files changed, 44 insertions(+), 7 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java index 02489147c65a..8e4ca7242d7e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java @@ -83,7 +83,14 @@ public String getEventType() { @Override public String getEventDescription() { - return "Deleting Kubernetes supported version " + getId(); + String description = "Deleting Kubernetes supported version"; + KubernetesSupportedVersion version = _entityMgr.findById(KubernetesSupportedVersion.class, getId()); + if (version != null) { + description += String.format(" ID: %s", version.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 32b07c4c36a4..54e307c0c5b9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -259,7 +259,7 @@ public String getCreateEventDescription() { @Override public String getEventDescription() { - return "creating Kubernetes cluster. Cluster Id: " + getEntityId(); + return "Creating Kubernetes cluster. Cluster Id: " + getEntityId(); } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java index 4f32138758ea..d6bc75c9c3d9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java @@ -102,8 +102,14 @@ public String getEventType() { @Override public String getEventDescription() { + String description = "Deleting Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Deleting Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index 90ccfa419175..994b39970222 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -94,8 +94,14 @@ public String getEventType() { @Override public String getEventDescription() { + String description = "Scaling Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Scaling Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java index 1ce2fe09c10f..d61a942e7f9f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java @@ -73,8 +73,14 @@ public String getEventType() { @Override public String getEventDescription() { + String description = "Starting Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Starting Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java index ba2649f863e0..d4f49bd225f3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java @@ -74,8 +74,14 @@ public String getEventType() { @Override public String getEventDescription() { + String description = "Stopping Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Stopping Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java index 2c99b005ff46..4a2e0b1c9a80 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java @@ -84,8 +84,14 @@ public String getEventType() { @Override public String getEventDescription() { + String description = "Upgrading Kubernetes cluster"; KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); - return String.format("Upgrading Kubernetes cluster ID: %s", cluster.getUuid()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; } @Override From 4b4dceaa623aea775af57e494578ef9b656f67fc Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 17 Sep 2020 01:08:47 +0530 Subject: [PATCH 24/27] improvements Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index f2ec874fa20b..f297c16b2914 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -575,7 +575,7 @@ def stopKubernetesCluster(self, cluster_id): def startKubernetesCluster(self, cluster_id): startKubernetesClusterCmd = startKubernetesCluster.startKubernetesClusterCmd() startKubernetesClusterCmd.id = cluster_id - response = self.apiclient.startnetesCluster(startKubernetesClusterCmd) + response = self.apiclient.startKubernetesCluster(startKubernetesClusterCmd) return response def deleteKubernetesCluster(self, cluster_id): @@ -610,6 +610,12 @@ def getValidKubernetesCluster(self, size=1, master_nodes=1): if valid == True and cluster.id == None: valid = False self.debug("ID for existing cluster not found, k8s_cluster ID: %s" % cluster.id) + if valid == True: + cluster_id = cluster.id + cluster = self.listKubernetesCluster(cluster_id) + if cluster == None: + valid = False + self.debug("Existing cluster, k8s_cluster ID: %s not returned by list API" % cluster_id) if valid == True: try: self.verifyKubernetesCluster(cluster, cluster.name, None, size, master_nodes) @@ -617,12 +623,17 @@ def getValidKubernetesCluster(self, size=1, master_nodes=1): except AssertionError as error: valid = False self.debug("Existing cluster failed verification due to %s, need to deploy a new one" % error) - self.deleteKubernetesClusterAndVerify(cluster.id, False, True) if valid == False: name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) - cluster = self.createKubernetesCluster(name, version.id, size, master_nodes) - self.verifyKubernetesCluster(cluster, name, version.id, size, master_nodes) + try: + self.deleteAllLeftoverClusters() + cluster = self.createKubernetesCluster(name, version.id, size, master_nodes) + self.verifyKubernetesCluster(cluster, name, version.id, size, master_nodes) + except Exception as ex: + self.fail("Kubernetes cluster deployment failed: %s" % ex) + except AssertionError as err: + self.fail("Kubernetes cluster deployment failed during cluster verification: %s" % err) return cluster def verifyKubernetesCluster(self, cluster_response, name, version_id=None, size=1, master_nodes=1): @@ -762,3 +773,9 @@ def deleteKubernetesClusterAndVerify(self, cluster_id, verify = True, forced = F None, "KubernetesCluster not removed in DB, {}".format(db_cluster_removed) ) + + def deleteAllLeftoverClusters(self): + clusters = self.listKubernetesCluster() + if clusters != None: + for cluster in clusters: + self.deleteKubernetesClusterAndVerify(cluster.id, False, True) From d29d177c0833f06d6c9a9028d6ab2d3d86cc42ef Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 17 Sep 2020 12:42:36 +0530 Subject: [PATCH 25/27] fix Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index f297c16b2914..2b7638d5324a 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -379,6 +379,7 @@ def test_03_deploy_and_upgrade_kubernetes_cluster(self): self.fail("Setup incomplete") global k8s_cluster k8s_cluster = self.getValidKubernetesCluster() + time.sleep(self.services["sleep"]) self.debug("Upgrading Kubernetes cluster with ID: %s" % k8s_cluster.id) @@ -506,7 +507,9 @@ def test_08_deploy_and_upgrade_kubernetes_ha_cluster(self): self.fail("Setup incomplete") global k8s_cluster k8s_cluster = self.getValidKubernetesCluster(1, 2) + time.sleep(self.services["sleep"]) + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) try: k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_4.id) except Exception as e: From 96aa16f4baa22cabc9fa991293618f030b9f9f1b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 21 Sep 2020 21:04:39 +0530 Subject: [PATCH 26/27] temp change to mention version Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterStartWorker.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 1ef5d0c788cd..e8b4ec2cdf6c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -161,6 +161,7 @@ private String getKubernetesMasterConfig(final String masterIp, final String ser KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); + initArgs += String.format(" --kubernetes-version=%s", kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()).getSemanticVersion()); k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); k8sMasterConfig = k8sMasterConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); return k8sMasterConfig; From 7e90a052229528185b18817f677a3748d266f6d1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 22 Sep 2020 10:50:10 +0530 Subject: [PATCH 27/27] changes for specifying k8s version Signed-off-by: Abhishek Kumar --- .../actionworkers/KubernetesClusterStartWorker.java | 13 +++++++++++-- scripts/util/create-kubernetes-binaries-iso.sh | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index e8b4ec2cdf6c..90c037512798 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -76,10 +76,19 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { + private KubernetesSupportedVersion kubernetesClusterVersion; + public KubernetesClusterStartWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); } + public KubernetesSupportedVersion getKubernetesClusterVersion() { + if (kubernetesClusterVersion == null) { + kubernetesClusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + } + return kubernetesClusterVersion; + } + private Pair> getKubernetesMasterIpAddresses(final DataCenter zone, final Network network, final Account account) throws InsufficientAddressCapacityException { String masterIp = null; Map requestedIps = null; @@ -105,7 +114,7 @@ private Pair> getKubernetesMasterIpAddres private boolean isKubernetesVersionSupportsHA() { boolean haSupported = false; - final KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + KubernetesSupportedVersion version = getKubernetesClusterVersion(); if (version != null) { try { if (KubernetesVersionManagerImpl.compareSemanticVersions(version.getSemanticVersion(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { @@ -161,7 +170,7 @@ private String getKubernetesMasterConfig(final String masterIp, final String ser KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); - initArgs += String.format(" --kubernetes-version=%s", kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()).getSemanticVersion()); + initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); k8sMasterConfig = k8sMasterConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); return k8sMasterConfig; diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index bf97f0662f7c..d7d9c16310d9 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -86,7 +86,7 @@ if [ $? -ne 0 ]; then fi fi mkdir -p "${working_dir}/docker" -output=`${k8s_dir}/kubeadm config images list` +output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` while read -r line; do echo "Downloading docker image $line ---" sudo docker pull "$line"