Skip to content

Commit

Permalink
Adding airgap tests to be run with tfp created infra
Browse files Browse the repository at this point in the history
  • Loading branch information
anupama2501 committed Mar 3, 2025
1 parent 2dee4ee commit 5dea5e2
Show file tree
Hide file tree
Showing 7 changed files with 199 additions and 95 deletions.
File renamed without changes.
47 changes: 47 additions & 0 deletions validation/airgap/airgap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package airgap

import (
"testing"

"github.com/rancher/shepherd/clients/corral"
"github.com/rancher/tests/validation/pipeline/rancherha/corralha"
"github.com/rancher/tests/validation/provisioning/registries"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
)

const (
corralPackageAirgapCustomClusterName = "airgapCustomCluster"
corralBastionIP = "bastion_ip"
corralRegistryIP = "registry_ip"
corralRegistryFQDN = "registry_fqdn"
logMessageKubernetesVersion = "Validating the current version is the upgraded one"
)

func airgapCorral(t *testing.T, corralRancherHA *corralha.CorralRancherHA) (registryFQDN string) {

listOfCorrals, err := corral.ListCorral()
require.NoError(t, err)
_, corralExist := listOfCorrals[corralRancherHA.Name]
registriesConfig := new(registries.Registries)

if corralExist {
bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryIP)
require.NoError(t, err)

err = corral.UpdateCorralConfig(corralBastionIP, bastionIP)
require.NoError(t, err)

registryFQDN, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryFQDN)
require.NoError(t, err)
logrus.Infof("registry fqdn is %s", registryFQDN)

err = corral.SetCorralSSHKeys(corralRancherHA.Name)
require.NoError(t, err)

return registryFQDN
} else {
registryFQDN = registriesConfig.ExistingNoAuthRegistryURL
}
return registryFQDN
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"github.com/rancher/tests/actions/reports"
"github.com/rancher/tests/validation/pipeline/rancherha/corralha"
"github.com/rancher/tests/validation/provisioning/registries"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
Expand All @@ -50,9 +49,6 @@ func (a *AirGapK3SCustomClusterTestSuite) SetupSuite() {
a.clustersConfig = new(provisioninginput.Config)
config.LoadConfig(provisioninginput.ConfigurationFileKey, a.clustersConfig)

corralRancherHA := new(corralha.CorralRancherHA)
config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA)

registriesConfig := new(registries.Registries)
config.LoadConfig(registries.RegistriesConfigKey, registriesConfig)

Expand All @@ -78,34 +74,16 @@ func (a *AirGapK3SCustomClusterTestSuite) SetupSuite() {

a.client = standardUserClient

listOfCorrals, err := corral.ListCorral()
require.NoError(a.T(), err)

corralConfig := corral.Configurations()

err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath)
require.NoError(a.T(), err)

a.corralPackage = corral.PackagesConfig()

_, corralExist := listOfCorrals[corralRancherHA.Name]
if corralExist {
bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryIP)
require.NoError(a.T(), err)

err = corral.UpdateCorralConfig(corralBastionIP, bastionIP)
require.NoError(a.T(), err)

registryFQDN, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryFQDN)
require.NoError(a.T(), err)
logrus.Infof("registry fqdn is %s", registryFQDN)
corralRancherHA := new(corralha.CorralRancherHA)
config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA)
if corralRancherHA.Name == "" {
a.registryFQDN = airgapCorral(a.T(), corralRancherHA)
corralConfig := corral.Configurations()

err = corral.SetCorralSSHKeys(corralRancherHA.Name)
err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath)
require.NoError(a.T(), err)

a.registryFQDN = registryFQDN
} else {
a.registryFQDN = registriesConfig.ExistingNoAuthRegistryURL
a.corralPackage = corral.PackagesConfig()
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"github.com/rancher/tests/actions/reports"
"github.com/rancher/tests/validation/pipeline/rancherha/corralha"
"github.com/rancher/tests/validation/provisioning/registries"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
Expand All @@ -48,9 +47,6 @@ func (a *AirGapRKE1CustomClusterTestSuite) SetupSuite() {
a.clustersConfig = new(provisioninginput.Config)
config.LoadConfig(provisioninginput.ConfigurationFileKey, a.clustersConfig)

corralRancherHA := new(corralha.CorralRancherHA)
config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA)

registriesConfig := new(registries.Registries)
config.LoadConfig(registries.RegistriesConfigKey, registriesConfig)

Expand All @@ -76,35 +72,17 @@ func (a *AirGapRKE1CustomClusterTestSuite) SetupSuite() {

a.client = standardUserClient

listOfCorrals, err := corral.ListCorral()
require.NoError(a.T(), err)

corralConfig := corral.Configurations()

err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath)
require.NoError(a.T(), err)

a.corralPackage = corral.PackagesConfig()

_, corralExist := listOfCorrals[corralRancherHA.Name]
if corralExist {
bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryIP)
require.NoError(a.T(), err)

err = corral.UpdateCorralConfig(corralBastionIP, bastionIP)
require.NoError(a.T(), err)
corralRancherHA := new(corralha.CorralRancherHA)
config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA)
if corralRancherHA.Name == "" {
a.registryFQDN = airgapCorral(a.T(), corralRancherHA)
corralConfig := corral.Configurations()

registryFQDN, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryFQDN)
err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath)
require.NoError(a.T(), err)
logrus.Infof("registry fqdn is %s", registryFQDN)

err = corral.SetCorralSSHKeys(corralRancherHA.Name)
require.NoError(a.T(), err)
a.registryFQDN = registryFQDN
} else {
a.registryFQDN = registriesConfig.ExistingNoAuthRegistryURL
a.corralPackage = corral.PackagesConfig()
}

}

func (a *AirGapRKE1CustomClusterTestSuite) TestProvisioningAirGapRKE1CustomCluster() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"github.com/rancher/tests/actions/reports"
"github.com/rancher/tests/validation/pipeline/rancherha/corralha"
"github.com/rancher/tests/validation/provisioning/registries"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
Expand All @@ -50,9 +49,6 @@ func (a *AirGapRKE2CustomClusterTestSuite) SetupSuite() {
a.clustersConfig = new(provisioninginput.Config)
config.LoadConfig(provisioninginput.ConfigurationFileKey, a.clustersConfig)

corralRancherHA := new(corralha.CorralRancherHA)
config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA)

registriesConfig := new(registries.Registries)
config.LoadConfig(registries.RegistriesConfigKey, registriesConfig)

Expand All @@ -78,33 +74,16 @@ func (a *AirGapRKE2CustomClusterTestSuite) SetupSuite() {

a.client = standardUserClient

listOfCorrals, err := corral.ListCorral()
require.NoError(a.T(), err)

corralConfig := corral.Configurations()
err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath)
require.NoError(a.T(), err)

a.corralPackage = corral.PackagesConfig()

_, corralExist := listOfCorrals[corralRancherHA.Name]
if corralExist {
bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryIP)
require.NoError(a.T(), err)

err = corral.UpdateCorralConfig(corralBastionIP, bastionIP)
require.NoError(a.T(), err)

registryFQDN, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryFQDN)
require.NoError(a.T(), err)
logrus.Infof("registry fqdn is %s", registryFQDN)
corralRancherHA := new(corralha.CorralRancherHA)
config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA)
if corralRancherHA.Name == "" {
a.registryFQDN = airgapCorral(a.T(), corralRancherHA)
corralConfig := corral.Configurations()

err = corral.SetCorralSSHKeys(corralRancherHA.Name)
err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath)
require.NoError(a.T(), err)

a.registryFQDN = registryFQDN
} else {
a.registryFQDN = registriesConfig.ExistingNoAuthRegistryURL
a.corralPackage = corral.PackagesConfig()
}
}

Expand Down
131 changes: 131 additions & 0 deletions validation/pipeline/airgap/Jenkinsfile.airgap.tests
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
#!groovy
node {
def homePath = pwd() + "/"
def rootPath = "/root/go/src/github.com/rancher/tfp-automation/"
def TERRAFORM_DIR = './root/go/src/github.com/rancher/tfp-automation/modules/airgap'
def testsDir = "github.com/rancher/tfp-automation/tests/${env.TEST_PACKAGE}"
def filename = "config.yml"
def configPath = ""
def job_name = "${JOB_NAME}"
if (job_name.contains('/')) {
job_names = job_name.split('/')
job_name = job_names[job_names.size() - 1]
}
def testContainer = "${job_name}${env.BUILD_NUMBER}_test"
def imageName = "tfp-automation-validation-${job_name}${env.BUILD_NUMBER}"
def testResultsOut = "results.xml"
def testResultsJSON = "results.json"
def envFile = ".env"
def config = env.CONFIG
def runFlag = "AirGap"
def rancherUrl = ""
def rancherPassword = ""
def privateRegistry = ""

def infraBranch = "${env.INFRA_BRANCH}"
if ("${env.INFRA_BRANCH}" != "null" && "${env.INFRA_BRANCH}" != "") {
branch = "${env.INFRA_BRANCH}"
}
def testBranch = "${env.TEST_BRANCH}"
if ("${env.TEST_BRANCH}" != "null" && "${env.TEST_BRANCH}" != "") {
branch = "${env.TEST_BRANCH}"
}
def infraRepo = scm.userRemoteConfigs
if ("${env.INFRA_REPO}" != "null" && "${env.INFRA_REPO}" != "") {
repo = [[url: "${env.INFRA_REPO}"]]
}
def testRepo = cm.userRemoteConfigs
if ("${env.TEST_REPO}" != "null" && "${env.TEST_REPO}" != "") {
repo = [[url: "${env.TEST_REPO}"]]
}
def timeout = "${env.TIMEOUT}"
if ("${env.TIMEOUT}" != "null" && "${env.TIMEOUT}" != "") {
timeout = "${env.TIMEOUT}"
}
def testPackage = "airgap/"
//def testCaseName = `-run \"TestWorkloadUpgradeTestSuite/TestWorkloadPreUpgrade\"`

withCredentials([ string(credentialsId: 'AWS_ACCESS_KEY_ID', variable: 'AWS_ACCESS_KEY_ID'),
string(credentialsId: 'AWS_SECRET_ACCESS_KEY', variable: 'AWS_SECRET_ACCESS_KEY'),
string(credentialsId: 'AWS_SSH_PEM_KEY', variable: 'AWS_SSH_PEM_KEY'),
string(credentialsId: 'AWS_SSH_KEY_NAME', variable: 'AWS_SSH_KEY_NAME')]){

stage('Checkout Infrastructure Repo') {
deleteDir()
checkout([
$class: 'GitSCM',
branches: [[name: "*/${infraBranch}"]],
extensions: scm.extensions + [[$class: 'CleanCheckout']],
userRemoteConfigs: infraRepo
])
}

stage('Configure and Build') {
config = config.replace('${AWS_SECRET_ACCESS_KEY}', env.AWS_SECRET_ACCESS_KEY)
config = config.replace('${AWS_ACCESS_KEY_ID}', env.AWS_ACCESS_KEY_ID)

writeFile file: filename, text: config

def decoded = new String(env.AWS_SSH_PEM_KEY.decodeBase64())
writeFile file: 'key.pem', text: decoded

env.CATTLE_TEST_CONFIG=rootPath+filename
configPath = "./rancher/tests/v2/validation/${filename}"

sh "./configure.sh"
sh "./build.sh"
}

stage('Setup') {
sh returnStdout: true, script: 'wget -qO ./yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64'
sh returnStdout:true, script: 'chmod a+x ./yq'
}

stage('Run Infrastructure Module Test') {
try {
sh """
docker run --name ${testContainer} -t -v ${homePath}key.pem:${rootPath}key.pem --env-file ${envFile} -v ${homePath}/tfp_output.log:/root/tfp_output.log ${imageName} sh -c "
/root/go/bin/gotestsum --format standard-verbose --packages=${testsDir} --junitfile ${testResultsOut} --jsonfile ${testResultsJSON} -- -timeout=${timeout} -v ${params.TEST_CASE};
if [ -f ${rootPath}reporter ]; then ${rootPath}reporter; fi"
"""
sh "docker cp ${testContainer}:/root/ ./"

rancherPassword = sh(script: "yq '.rancher_url' tfpOutput", returnStdout: true).trim()
rancherUrl = sh(script: "awk -F'URL: ' '/Rancher server URL:/ {print \$2}' tfp_output.log", returnStdout: true).trim()
rancherPassword = sh(script: "awk -F'password: ' '/Booststrap password:/ {print \$2}' tfp_output.log", returnStdout: true).trim()
privateRegistry = sh(script: "awk -F'registry: ' '/Private registry:/ {print \$2}' tfp_output.log", returnStdout: true).trim()

} catch(err) {
echo 'Test run had failures. Collecting results...'
}
}

stage('Checkout Test Repo') {
checkout([
$class: 'GitSCM',
branches: [[name: "*/${testBranch}"]],
extensions: scm.extensions + [[$class: 'CleanCheckout']],
userRemoteConfigs: testRepo
])
}

stage('Run Downstream cluster tests'){
def configYaml = readFile(file: "${configPath}")
def goValidationParams = [
text(name: 'CONFIG', value: configYaml),
string(name: 'BRANCH', value: "main"),
string(name: 'TEST_RUN_NAME', value: ${testPackage}),
string(name: 'GOTEST_TESTCASE', value: "${[runFlag]} -p 1"),
]

jobs["$testTag-test-validation"] = { build job: 'go-automation-freeform-job', parameters: goValidationParams }

}

stage('Cleanup terraform resources'){
sh """
docker exec $testContainer sh -c "chmod +x -R /terraform-files && terraform -chdir=/terraform-files init && terraform -chdir=/terraform-files destroy -auto-approve"
"""
} //cleanup
} //credentials
} // node
9 changes: 0 additions & 9 deletions validation/provisioning/airgap/airgap.go

This file was deleted.

0 comments on commit 5dea5e2

Please sign in to comment.