diff --git a/validation/provisioning/airgap/README.md b/validation/airgap/README.md similarity index 100% rename from validation/provisioning/airgap/README.md rename to validation/airgap/README.md diff --git a/validation/airgap/airgap.go b/validation/airgap/airgap.go new file mode 100644 index 0000000000..10ed01b6f0 --- /dev/null +++ b/validation/airgap/airgap.go @@ -0,0 +1,47 @@ +package airgap + +import ( + "testing" + + "github.com/rancher/shepherd/clients/corral" + "github.com/rancher/tests/validation/pipeline/rancherha/corralha" + "github.com/rancher/tests/validation/provisioning/registries" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +const ( + corralPackageAirgapCustomClusterName = "airgapCustomCluster" + corralBastionIP = "bastion_ip" + corralRegistryIP = "registry_ip" + corralRegistryFQDN = "registry_fqdn" + logMessageKubernetesVersion = "Validating the current version is the upgraded one" +) + +func airgapCorral(t *testing.T, corralRancherHA *corralha.CorralRancherHA) (registryFQDN string) { + + listOfCorrals, err := corral.ListCorral() + require.NoError(t, err) + _, corralExist := listOfCorrals[corralRancherHA.Name] + registriesConfig := new(registries.Registries) + + if corralExist { + bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryIP) + require.NoError(t, err) + + err = corral.UpdateCorralConfig(corralBastionIP, bastionIP) + require.NoError(t, err) + + registryFQDN, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryFQDN) + require.NoError(t, err) + logrus.Infof("registry fqdn is %s", registryFQDN) + + err = corral.SetCorralSSHKeys(corralRancherHA.Name) + require.NoError(t, err) + + return registryFQDN + } else { + registryFQDN = registriesConfig.ExistingNoAuthRegistryURL + } + return registryFQDN +} diff --git a/validation/provisioning/airgap/k3s_custom_cluster_test.go b/validation/airgap/k3s_custom_cluster_test.go similarity index 88% rename from validation/provisioning/airgap/k3s_custom_cluster_test.go rename to validation/airgap/k3s_custom_cluster_test.go index a36c8ebde2..2f6593589f 100644 --- a/validation/provisioning/airgap/k3s_custom_cluster_test.go +++ b/validation/airgap/k3s_custom_cluster_test.go @@ -24,7 +24,6 @@ import ( "github.com/rancher/tests/actions/reports" "github.com/rancher/tests/validation/pipeline/rancherha/corralha" "github.com/rancher/tests/validation/provisioning/registries" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -50,9 +49,6 @@ func (a *AirGapK3SCustomClusterTestSuite) SetupSuite() { a.clustersConfig = new(provisioninginput.Config) config.LoadConfig(provisioninginput.ConfigurationFileKey, a.clustersConfig) - corralRancherHA := new(corralha.CorralRancherHA) - config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA) - registriesConfig := new(registries.Registries) config.LoadConfig(registries.RegistriesConfigKey, registriesConfig) @@ -78,34 +74,16 @@ func (a *AirGapK3SCustomClusterTestSuite) SetupSuite() { a.client = standardUserClient - listOfCorrals, err := corral.ListCorral() - require.NoError(a.T(), err) - - corralConfig := corral.Configurations() - - err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath) - require.NoError(a.T(), err) - - a.corralPackage = corral.PackagesConfig() - - _, corralExist := listOfCorrals[corralRancherHA.Name] - if corralExist { - bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryIP) - require.NoError(a.T(), err) - - err = corral.UpdateCorralConfig(corralBastionIP, bastionIP) - require.NoError(a.T(), err) - - registryFQDN, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryFQDN) - require.NoError(a.T(), err) - logrus.Infof("registry fqdn is %s", registryFQDN) + corralRancherHA := new(corralha.CorralRancherHA) + config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA) + if corralRancherHA.Name == "" { + a.registryFQDN = airgapCorral(a.T(), corralRancherHA) + corralConfig := corral.Configurations() - err = corral.SetCorralSSHKeys(corralRancherHA.Name) + err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath) require.NoError(a.T(), err) - a.registryFQDN = registryFQDN - } else { - a.registryFQDN = registriesConfig.ExistingNoAuthRegistryURL + a.corralPackage = corral.PackagesConfig() } } diff --git a/validation/provisioning/airgap/rke1_custom_cluster_test.go b/validation/airgap/rke1_custom_cluster_test.go similarity index 87% rename from validation/provisioning/airgap/rke1_custom_cluster_test.go rename to validation/airgap/rke1_custom_cluster_test.go index c0e35ebaf2..2117cb07a1 100644 --- a/validation/provisioning/airgap/rke1_custom_cluster_test.go +++ b/validation/airgap/rke1_custom_cluster_test.go @@ -22,7 +22,6 @@ import ( "github.com/rancher/tests/actions/reports" "github.com/rancher/tests/validation/pipeline/rancherha/corralha" "github.com/rancher/tests/validation/provisioning/registries" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -48,9 +47,6 @@ func (a *AirGapRKE1CustomClusterTestSuite) SetupSuite() { a.clustersConfig = new(provisioninginput.Config) config.LoadConfig(provisioninginput.ConfigurationFileKey, a.clustersConfig) - corralRancherHA := new(corralha.CorralRancherHA) - config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA) - registriesConfig := new(registries.Registries) config.LoadConfig(registries.RegistriesConfigKey, registriesConfig) @@ -76,35 +72,17 @@ func (a *AirGapRKE1CustomClusterTestSuite) SetupSuite() { a.client = standardUserClient - listOfCorrals, err := corral.ListCorral() - require.NoError(a.T(), err) - - corralConfig := corral.Configurations() - - err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath) - require.NoError(a.T(), err) - - a.corralPackage = corral.PackagesConfig() - - _, corralExist := listOfCorrals[corralRancherHA.Name] - if corralExist { - bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryIP) - require.NoError(a.T(), err) - - err = corral.UpdateCorralConfig(corralBastionIP, bastionIP) - require.NoError(a.T(), err) + corralRancherHA := new(corralha.CorralRancherHA) + config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA) + if corralRancherHA.Name == "" { + a.registryFQDN = airgapCorral(a.T(), corralRancherHA) + corralConfig := corral.Configurations() - registryFQDN, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryFQDN) + err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath) require.NoError(a.T(), err) - logrus.Infof("registry fqdn is %s", registryFQDN) - err = corral.SetCorralSSHKeys(corralRancherHA.Name) - require.NoError(a.T(), err) - a.registryFQDN = registryFQDN - } else { - a.registryFQDN = registriesConfig.ExistingNoAuthRegistryURL + a.corralPackage = corral.PackagesConfig() } - } func (a *AirGapRKE1CustomClusterTestSuite) TestProvisioningAirGapRKE1CustomCluster() { diff --git a/validation/provisioning/airgap/rke2_custom_cluster_test.go b/validation/airgap/rke2_custom_cluster_test.go similarity index 88% rename from validation/provisioning/airgap/rke2_custom_cluster_test.go rename to validation/airgap/rke2_custom_cluster_test.go index 60ad03e7c2..7a1b079dc1 100644 --- a/validation/provisioning/airgap/rke2_custom_cluster_test.go +++ b/validation/airgap/rke2_custom_cluster_test.go @@ -24,7 +24,6 @@ import ( "github.com/rancher/tests/actions/reports" "github.com/rancher/tests/validation/pipeline/rancherha/corralha" "github.com/rancher/tests/validation/provisioning/registries" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -50,9 +49,6 @@ func (a *AirGapRKE2CustomClusterTestSuite) SetupSuite() { a.clustersConfig = new(provisioninginput.Config) config.LoadConfig(provisioninginput.ConfigurationFileKey, a.clustersConfig) - corralRancherHA := new(corralha.CorralRancherHA) - config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA) - registriesConfig := new(registries.Registries) config.LoadConfig(registries.RegistriesConfigKey, registriesConfig) @@ -78,33 +74,16 @@ func (a *AirGapRKE2CustomClusterTestSuite) SetupSuite() { a.client = standardUserClient - listOfCorrals, err := corral.ListCorral() - require.NoError(a.T(), err) - - corralConfig := corral.Configurations() - err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath) - require.NoError(a.T(), err) - - a.corralPackage = corral.PackagesConfig() - - _, corralExist := listOfCorrals[corralRancherHA.Name] - if corralExist { - bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryIP) - require.NoError(a.T(), err) - - err = corral.UpdateCorralConfig(corralBastionIP, bastionIP) - require.NoError(a.T(), err) - - registryFQDN, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryFQDN) - require.NoError(a.T(), err) - logrus.Infof("registry fqdn is %s", registryFQDN) + corralRancherHA := new(corralha.CorralRancherHA) + config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA) + if corralRancherHA.Name == "" { + a.registryFQDN = airgapCorral(a.T(), corralRancherHA) + corralConfig := corral.Configurations() - err = corral.SetCorralSSHKeys(corralRancherHA.Name) + err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath) require.NoError(a.T(), err) - a.registryFQDN = registryFQDN - } else { - a.registryFQDN = registriesConfig.ExistingNoAuthRegistryURL + a.corralPackage = corral.PackagesConfig() } } diff --git a/validation/pipeline/airgap/Jenkinsfile.airgap.tests b/validation/pipeline/airgap/Jenkinsfile.airgap.tests new file mode 100644 index 0000000000..d478bdefde --- /dev/null +++ b/validation/pipeline/airgap/Jenkinsfile.airgap.tests @@ -0,0 +1,131 @@ +#!groovy +node { + def homePath = pwd() + "/" + def rootPath = "/root/go/src/github.com/rancher/tfp-automation/" + def TERRAFORM_DIR = './root/go/src/github.com/rancher/tfp-automation/modules/airgap' + def testsDir = "github.com/rancher/tfp-automation/tests/${env.TEST_PACKAGE}" + def filename = "config.yml" + def configPath = "" + def job_name = "${JOB_NAME}" + if (job_name.contains('/')) { + job_names = job_name.split('/') + job_name = job_names[job_names.size() - 1] + } + def testContainer = "${job_name}${env.BUILD_NUMBER}_test" + def imageName = "tfp-automation-validation-${job_name}${env.BUILD_NUMBER}" + def testResultsOut = "results.xml" + def testResultsJSON = "results.json" + def envFile = ".env" + def config = env.CONFIG + def runFlag = "AirGap" + def rancherUrl = "" + def rancherPassword = "" + def privateRegistry = "" + + def infraBranch = "${env.INFRA_BRANCH}" + if ("${env.INFRA_BRANCH}" != "null" && "${env.INFRA_BRANCH}" != "") { + branch = "${env.INFRA_BRANCH}" + } + def testBranch = "${env.TEST_BRANCH}" + if ("${env.TEST_BRANCH}" != "null" && "${env.TEST_BRANCH}" != "") { + branch = "${env.TEST_BRANCH}" + } + def infraRepo = scm.userRemoteConfigs + if ("${env.INFRA_REPO}" != "null" && "${env.INFRA_REPO}" != "") { + repo = [[url: "${env.INFRA_REPO}"]] + } + def testRepo = cm.userRemoteConfigs + if ("${env.TEST_REPO}" != "null" && "${env.TEST_REPO}" != "") { + repo = [[url: "${env.TEST_REPO}"]] + } + def timeout = "${env.TIMEOUT}" + if ("${env.TIMEOUT}" != "null" && "${env.TIMEOUT}" != "") { + timeout = "${env.TIMEOUT}" + } + def testPackage = "airgap/" + //def testCaseName = `-run \"TestWorkloadUpgradeTestSuite/TestWorkloadPreUpgrade\"` + + withCredentials([ string(credentialsId: 'AWS_ACCESS_KEY_ID', variable: 'AWS_ACCESS_KEY_ID'), + string(credentialsId: 'AWS_SECRET_ACCESS_KEY', variable: 'AWS_SECRET_ACCESS_KEY'), + string(credentialsId: 'AWS_SSH_PEM_KEY', variable: 'AWS_SSH_PEM_KEY'), + string(credentialsId: 'AWS_SSH_KEY_NAME', variable: 'AWS_SSH_KEY_NAME')]){ + + stage('Checkout Infrastructure Repo') { + deleteDir() + checkout([ + $class: 'GitSCM', + branches: [[name: "*/${infraBranch}"]], + extensions: scm.extensions + [[$class: 'CleanCheckout']], + userRemoteConfigs: infraRepo + ]) + } + + stage('Configure and Build') { + config = config.replace('${AWS_SECRET_ACCESS_KEY}', env.AWS_SECRET_ACCESS_KEY) + config = config.replace('${AWS_ACCESS_KEY_ID}', env.AWS_ACCESS_KEY_ID) + + writeFile file: filename, text: config + + def decoded = new String(env.AWS_SSH_PEM_KEY.decodeBase64()) + writeFile file: 'key.pem', text: decoded + + env.CATTLE_TEST_CONFIG=rootPath+filename + configPath = "./rancher/tests/v2/validation/${filename}" + + sh "./configure.sh" + sh "./build.sh" + } + + stage('Setup') { + sh returnStdout: true, script: 'wget -qO ./yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64' + sh returnStdout:true, script: 'chmod a+x ./yq' + } + + stage('Run Infrastructure Module Test') { + try { + sh """ + docker run --name ${testContainer} -t -v ${homePath}key.pem:${rootPath}key.pem --env-file ${envFile} -v ${homePath}/tfp_output.log:/root/tfp_output.log ${imageName} sh -c " + /root/go/bin/gotestsum --format standard-verbose --packages=${testsDir} --junitfile ${testResultsOut} --jsonfile ${testResultsJSON} -- -timeout=${timeout} -v ${params.TEST_CASE}; + if [ -f ${rootPath}reporter ]; then ${rootPath}reporter; fi" + """ + sh "docker cp ${testContainer}:/root/ ./" + + rancherPassword = sh(script: "yq '.rancher_url' tfpOutput", returnStdout: true).trim() + rancherUrl = sh(script: "awk -F'URL: ' '/Rancher server URL:/ {print \$2}' tfp_output.log", returnStdout: true).trim() + rancherPassword = sh(script: "awk -F'password: ' '/Booststrap password:/ {print \$2}' tfp_output.log", returnStdout: true).trim() + privateRegistry = sh(script: "awk -F'registry: ' '/Private registry:/ {print \$2}' tfp_output.log", returnStdout: true).trim() + + } catch(err) { + echo 'Test run had failures. Collecting results...' + } + } + + stage('Checkout Test Repo') { + checkout([ + $class: 'GitSCM', + branches: [[name: "*/${testBranch}"]], + extensions: scm.extensions + [[$class: 'CleanCheckout']], + userRemoteConfigs: testRepo + ]) + } + + stage('Run Downstream cluster tests'){ + def configYaml = readFile(file: "${configPath}") + def goValidationParams = [ + text(name: 'CONFIG', value: configYaml), + string(name: 'BRANCH', value: "main"), + string(name: 'TEST_RUN_NAME', value: ${testPackage}), + string(name: 'GOTEST_TESTCASE', value: "${[runFlag]} -p 1"), + ] + + jobs["$testTag-test-validation"] = { build job: 'go-automation-freeform-job', parameters: goValidationParams } + + } + + stage('Cleanup terraform resources'){ + sh """ + docker exec $testContainer sh -c "chmod +x -R /terraform-files && terraform -chdir=/terraform-files init && terraform -chdir=/terraform-files destroy -auto-approve" + """ + } //cleanup + } //credentials +} // node \ No newline at end of file diff --git a/validation/provisioning/airgap/airgap.go b/validation/provisioning/airgap/airgap.go deleted file mode 100644 index 22d0d92f0b..0000000000 --- a/validation/provisioning/airgap/airgap.go +++ /dev/null @@ -1,9 +0,0 @@ -package airgap - -const ( - corralPackageAirgapCustomClusterName = "airgapCustomCluster" - corralBastionIP = "bastion_ip" - corralRegistryIP = "registry_ip" - corralRegistryFQDN = "registry_fqdn" - logMessageKubernetesVersion = "Validating the current version is the upgraded one" -)