From 08d5fff4bd2db1d480214c81bdb69abc7eb7c57b Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 7 Oct 2021 14:27:37 +0530 Subject: [PATCH] mbx: Automate Storage management - SysVM template download & seeding (#19) mbx launch now creates systemvmtemplate cache and seeds template to secondary storage if necessary before launching zone Signed-off-by: Rohit Yadav Co-authored-by: Rohit Yadav --- .gitignore | 1 + README.md | 88 +++++++-------------------------- files/setup-systemvmtemplate.sh | 3 ++ files/sudoer.mbx | 2 +- files/systemvm-map.json | 86 ++++++++++++++++++++++++++++++++ mbx | 86 ++++++++++++++++++++------------ templates/cache/.gitkeep | 0 7 files changed, 164 insertions(+), 102 deletions(-) create mode 100644 files/systemvm-map.json create mode 100644 templates/cache/.gitkeep diff --git a/.gitignore b/.gitignore index dca8823..0104a8d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ boxes/boxes.list boxes/** templates/*.qcow2 +templates/cache/* diff --git a/README.md b/README.md index 7b25d8d..9b3e673 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ Table of Contents * [Setup NFS Storage](#setup-nfs-storage) * [Setup KVM](#setup-kvm) * [Setup mbx](#setup-mbx) - * [Create Storage Gold-Masters](#create-storage-gold-masters) * [Using mbx](#using-mbx) * [CloudStack Development](#cloudstack-development) * [Install Development Tools](#install-development-tools) @@ -40,7 +39,9 @@ nested guest VMs and virtual router are plugged in nested-virtual networks that run in a nested KVM host VM. To learn more about CloudStack as a user, you may read: + https://github.com/shapeblue/hackerbook/blob/main/0-init.md + https://github.com/shapeblue/hackerbook/blob/main/1-user.md ### Storage @@ -48,13 +49,13 @@ https://github.com/shapeblue/hackerbook/blob/main/1-user.md `mbx` requires NFS storage to be setup and exported for the base path `/export/testing` for environment-specific primary and secondary storages. -A typical `mbx` environment deployment makes copy of a CloudStack -version-specific gold-master directory that generally contains two empty primary -storage directories (`primary1` and `primary2`) and one secondary storage -directory (`secondary`). The secondary storage directory must be seeded -with CloudStack version-specific `systemvmtemplates`. The `systemvmtemplate` is -then used to create system VMs such as the Secondary-Storage VM, Console-Proxy -VM and Virtual Router in an `mbx` environment. +A typical `mbx` environment upon deployment creates two primary storage +directories (`primary1` and `primary2`) and one secondary storage directory +(`secondary`). The secondary storage directory is seeded with supported +CloudStack version-specific `systemvmtemplates` if necessary when the env is +launched using `mbx launch`. The seeded `systemvmtemplate` is then used to +create system VMs such as the Secondary-Storage VM, Console-Proxy VM and Virtual +Router in an `mbx` environment. ### Networking @@ -126,6 +127,8 @@ Tested CloudStack versions: - 4.15.2.0 - 4.16.0.0-SNAPSHOT (main branch) +Supported CloudStack versions: 4.9, 4.11, 4.12, 4.13, 4.14, 4.15 and later + Note: legacy CloudStack releases older than v4.11 that don't have `cloudstack-marvin` package will not work. @@ -255,59 +258,6 @@ NAT-enabled virtual network in 172.20.0.0/16. Your workstation/host IP address is `172.20.0.1`. -### Create Storage Gold-Masters - -Note: This is required to be done only once for a specific version of CloudStack and -is only required for CloudStack **4.15 or below**. - -After setting up NFS on the workstation host, you need to create a -CloudStack-version specific storage golden master directory that contains two -primary storage folders and a secondary storage folder with the systemvmtemplate for the -specific version of CloudStack seeded. The storage golden master is used as -storage source of an mbx environment during `mbx deploy` command execution. - -For example, the following is needed only one-time for creating a golden master -storage directory for CloudStack 4.14 version: - - mkdir -p /export/testing - # Create directory layout for a specific ACS version under /export/testing - mkdir -p /export/testing/4.14/{primary1,primary2,secondary} - # Get the systemvm templates - cd /export/testing/4.14 - wget http://packages.shapeblue.com/systemvmtemplate/4.14/systemvmtemplate-4.14.0-kvm.qcow2.bz2 - wget http://packages.shapeblue.com/systemvmtemplate/4.14/systemvmtemplate-4.14.0-vmware.ova - wget http://packages.shapeblue.com/systemvmtemplate/4.14/systemvmtemplate-4.14.0-xen.vhd.bz2 - wget http://packages.shapeblue.com/systemvmtemplate/4.14/md5sum.txt - # Check the downloaded templates, it should say OK for the three templates - md5sum --check md5sum.txt - # Seed template in the secondary folder for 4.14 - /export/monkeybox/files/setup-systemvmtemplate.sh -m /export/testing/4.14/secondary -f systemvmtemplate-4.14.0-kvm.qcow2.bz2 -h kvm - /export/monkeybox/files/setup-systemvmtemplate.sh -m /export/testing/4.14/secondary -f systemvmtemplate-4.14.0-vmware.ova -h vmware - /export/monkeybox/files/setup-systemvmtemplate.sh -m /export/testing/4.14/secondary -f systemvmtemplate-4.14.0-xen.vhd.bz2 -h xenserver - # Cleanup downloaded files - rm -fv md5sum.txt systemvmtemplate* - -For example, the following is needed only one-time for creating a golden master -storage directory for CloudStack 4.15 version: - - mkdir -p /export/testing - # Create directory layout for a specific ACS version under /export/testing - mkdir -p /export/testing/4.15/{primary1,primary2,secondary} - # Get the systemvm templates - cd /export/testing/4.15 - wget http://packages.shapeblue.com/systemvmtemplate/4.15/systemvmtemplate-4.15.1-kvm.qcow2.bz2 - wget http://packages.shapeblue.com/systemvmtemplate/4.15/systemvmtemplate-4.15.1-vmware.ova - wget http://packages.shapeblue.com/systemvmtemplate/4.15/systemvmtemplate-4.15.1-xen.vhd.bz2 - wget http://packages.shapeblue.com/systemvmtemplate/4.15/md5sum.txt - # Check the downloaded templates, it should say OK for the three templates - md5sum --check md5sum.txt - # Seed template in the secondary folder for 4.15 - /export/monkeybox/files/setup-systemvmtemplate.sh -m /export/testing/4.15/secondary -f systemvmtemplate-4.15.1-kvm.qcow2.bz2 -h kvm - /export/monkeybox/files/setup-systemvmtemplate.sh -m /export/testing/4.15/secondary -f systemvmtemplate-4.15.1-vmware.ova -h vmware - /export/monkeybox/files/setup-systemvmtemplate.sh -m /export/testing/4.15/secondary -f systemvmtemplate-4.15.1-xen.vhd.bz2 -h xenserver - # Cleanup downloaded files - rm -fv md5sum.txt systemvmtemplate* - ## Using `mbx` The `mbx` tool can be used to build CloudStack packages, deploy dev or QA @@ -315,22 +265,22 @@ environments with KVM, VMware, XenServer and XCP-ng hypervisors, and run smoketests on them. $ mbx - MonkeyBox 🐵 v0.1 + MonkeyBox 🐵 v0.2 Available commands are: init: initialises monkeynet and mbx templates package: builds packages from a git repo and sha/tag/branch list: lists available environments - deploy: deploys QA env with two monkeybox VMs, configures storage, creates marvin cfg file + deploy: creates QA env with two monkeybox VMs and creates marvin cfg file launch: launches QA env zone using environment's marvin cfg file test: start marvin tests - dev: deploys dev env with a single monkeybox VM, configures storage, creates marvin cfg file + dev: creates dev env with a single monkeybox VM and creates marvin cfg file agentscp: updates KVM agent in dev environment using scp and restarts it ssh: ssh into a mbx VM stop: stop all env VMs start: start all env VMs destroy: destroy environment -0. On first run, initialise networking and templates, run: +0. On first run or when upgrading `mbx`, please run: mbx init @@ -340,7 +290,7 @@ smoketests on them. 2. To deploy an environment, run: - mbx deploy + mbx deploy Example to deploy test matrix (kvm, vmware, xenserver) environments: @@ -348,9 +298,9 @@ Example to deploy test matrix (kvm, vmware, xenserver) environments: mbx deploy 415-venv mbxt-kvm-centos7 mbxt-vmware67u3 # deploys 4.15 + VMware67u3 env mbx deploy 415-xenv mbxt-kvm-centos7 mbxt-xenserver71 # deploys 4.15 + XenServer71 env -More examples with specific repositories and custom storage source: (custom storage source must exist) +More examples with custom packages repositories: - mbx deploy 416-snapshot mbxt-kvm-centos7 mbxt-kvm-centos7 http://download.cloudstack.org/testing/nightly/latest/centos7/4.16 /export/testing/4.16.0 + mbx deploy 416-snapshot mbxt-kvm-centos7 mbxt-kvm-centos7 https://download.cloudstack.org/centos/7/4.15/ 3. Once `mbx` environment is deployed, to launch a zone run: @@ -360,7 +310,7 @@ More examples with specific repositories and custom storage source: (custom stor mbx list # find your environment mbx ssh - cd /marvin # here you'll find smoketests.sh to run smoketests + cd /marvin bash -x smoketests.sh 5. To destroy your mbx environment, run: diff --git a/files/setup-systemvmtemplate.sh b/files/setup-systemvmtemplate.sh index dfb4c26..0b9436e 100755 --- a/files/setup-systemvmtemplate.sh +++ b/files/setup-systemvmtemplate.sh @@ -36,6 +36,9 @@ failed() { fi if [[ $returnval -eq 0 ]]; then return 0 + elif [[ -n "(echo $returnmsg | grep "already")" ]]; then + echo $returnmsg + exit 0 else echo "Installation failed" exit $returnval diff --git a/files/sudoer.mbx b/files/sudoer.mbx index 122a142..886d551 100644 --- a/files/sudoer.mbx +++ b/files/sudoer.mbx @@ -1,4 +1,4 @@ -Cmnd_Alias CLOUDSTACK = /bin/mkdir, /bin/mount, /bin/umount, /bin/cp, /bin/chmod, /usr/bin/keytool, /bin/keytool, /usr/bin/virsh, /usr/bin/rm -fr /export/testing/* +Cmnd_Alias CLOUDSTACK = /bin/mkdir, /bin/mount, /bin/umount, /bin/cp, /bin/chmod, /usr/bin/keytool, /bin/keytool, /usr/bin/virsh, /usr/bin/rm -fr /export/testing/*, /export/monkeybox/files/setup-systemvmtemplate.sh Defaults:%sudo !requiretty diff --git a/files/systemvm-map.json b/files/systemvm-map.json new file mode 100644 index 0000000..08fa167 --- /dev/null +++ b/files/systemvm-map.json @@ -0,0 +1,86 @@ +{ + "4.9": { + "kvm": { + "url": "https://download.cloudstack.org/systemvm/4.6/dnsmasq/systemvm64template-4.6-kvm.qcow2.bz2", + "checksum": "c713c00fcca7e01f089a1962237e1cd8" + }, + "xen": { + "url": "https://download.cloudstack.org/systemvm/4.6/dnsmasq/systemvm64template-4.6-xen.vhd.bz2", + "checksum": "80c072799ac094fddfb8cba9bd0305e9" + }, + "vmw": { + "url": "https://download.cloudstack.org/systemvm/4.6/dnsmasq/systemvm64template-4.6-vmware.ova", + "checksum": "d91212b6c96b98bfd1559ef2cc2891e1" + } + }, + "4.11": { + "kvm": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-kvm.qcow2.bz2", + "checksum": "d40bce40b2d5bb4ba73e56d1e95aeae5" + }, + "xen": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-xen.vhd.bz2", + "checksum": "1566dcbcc3806755d0012d1619bd4210" + }, + "vmw": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-vmware.ova", + "checksum": "d695376be20929d323adfaa5410c093f" + } + }, + "4.12": { + "kvm": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-kvm.qcow2.bz2", + "checksum": "d40bce40b2d5bb4ba73e56d1e95aeae5" + }, + "xen": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-xen.vhd.bz2", + "checksum": "1566dcbcc3806755d0012d1619bd4210" + }, + "vmw": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-vmware.ova", + "checksum": "d695376be20929d323adfaa5410c093f" + } + }, + "4.13": { + "kvm": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-kvm.qcow2.bz2", + "checksum": "d40bce40b2d5bb4ba73e56d1e95aeae5" + }, + "xen": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-xen.vhd.bz2", + "checksum": "1566dcbcc3806755d0012d1619bd4210" + }, + "vmw": { + "url": "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.3-vmware.ova", + "checksum": "d695376be20929d323adfaa5410c093f" + } + }, + "4.14": { + "kvm": { + "url": "https://download.cloudstack.org/systemvm/4.14/systemvmtemplate-4.14.0-kvm.qcow2.bz2", + "checksum": "4978e6e6140d167556f201496549a498" + }, + "xen": { + "url": "https://download.cloudstack.org/systemvm/4.14/systemvmtemplate-4.14.0-xen.vhd.bz2", + "checksum": "2e3078de2ccce760d537e06fd9b4c7c7" + }, + "vmw": { + "url": "https://download.cloudstack.org/systemvm/4.14/systemvmtemplate-4.14.0-vmware.ova", + "checksum": "33cad72f858aef11c95df486b0f21938" + } + }, + "4.15": { + "kvm": { + "url": "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-kvm.qcow2.bz2", + "checksum": "0e9f9a7d0957c3e0a2088e41b2da2cec" + }, + "xen": { + "url": "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-xen.vhd.bz2", + "checksum": "86373992740b1eca8aff8b08ebf3aea5" + }, + "vmw": { + "url": "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-vmware.ova", + "checksum": "4006982765846d373eb3719b2fe4d720" + } + } +} diff --git a/mbx b/mbx index 5a002f8..8795a51 100755 --- a/mbx +++ b/mbx @@ -21,17 +21,17 @@ ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" mkdir -p $ROOT/boxes touch $ROOT/boxes/boxes.list export LIBVIRT_DEFAULT_URI="qemu:///system" -echo MonkeyBox 🐵 v0.1 +echo MonkeyBox 🐵 v0.2 usage() { echo "Available commands are:" echo " init: initialises monkeynet and mbx templates" echo " package: builds packages from a git repo and sha/tag/branch" echo " list: lists available environments" - echo " deploy: deploys QA env with two monkeybox VMs, configures storage, creates marvin cfg file" + echo " deploy: creates QA env with two monkeybox VMs and creates marvin cfg file" echo " launch: launches QA env zone using environment's marvin cfg file" echo " test: start marvin tests" - echo " dev: deploys dev env with a single monkeybox VM, configures storage, creates marvin cfg file" + echo " dev: creates dev env with a single monkeybox VM and creates marvin cfg file" echo " agentscp: updates KVM agent in dev environment using scp and restarts it" echo " ssh: ssh into a mbx VM" echo " stop: stop all env VMs" @@ -157,22 +157,46 @@ check_mbxt() { } issh() { - sshpass -p 'P@ssword123' ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" $@ + sshpass -p 'P@ssword123' ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -o "LogLevel=ERROR" $@ } iscp() { - sshpass -p 'P@ssword123' scp -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" $@ + sshpass -p 'P@ssword123' scp -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -o "LogLevel=ERROR" $@ +} + +seed_systemvmtemplate() { + version=$1 + hypervisor=$2 + sstore_path=$3 + cache_path=/export/monkeybox/templates/cache/$version/$hypervisor + url=$(cat $ROOT/files/systemvm-map.json | jq -r ".[\"${version}\"].${hypervisor}.url") + checksum=$(cat $ROOT/files/systemvm-map.json | jq -r ".[\"${version}\"].${hypervisor}.checksum") + template_path=$cache_path/${url##*/} + mkdir -p $cache_path + wget -nc $url -O $template_path || true + if [[ "$(md5sum $template_path | awk '{print $1}')" != "$checksum" ]]; then + rm -f $template_path + echo "The systemvmtemplate cache is invalid, removed old cache, please re-run launch command" + exit 1 + fi + echo "Seeding v$version systemvmtemplate for '$hypervisor' at $sstore_path" + if [[ "$hypervisor" == "xen" || "$hypervisor" == "xcp" ]]; then + hypervisor="xenserver" + fi + if [[ "$hypervisor" == "vmw" ]]; then + hypervisor="vmware" + fi + sudo /export/monkeybox/files/setup-systemvmtemplate.sh -m $sstore_path -f $template_path -h $hypervisor -F } deploy() { if [[ "$1" == "-h" ]]; then - echo "Usage: mbx deploy " + echo "Usage: mbx deploy " exit 0 fi - set -x id= - for idx in {1..9}; do + for idx in {1..20}; do if ! grep -q qa$idx- $ROOT/boxes/boxes.list; then id=$idx break @@ -183,27 +207,24 @@ deploy() { exit 1 fi - name=${1:-"mbxe"} + name=${1:-"mbx-cs415"} mst=${2:-"mbxt-kvm-centos7"} hyt=${3:-"mbxt-kvm-centos7"} - # Validate input templates + # Validate templates check_mbxt mgmt $mst check_mbxt hypervisor $hyt - hypervisor=$(echo $hyt | awk '{print substr($0,6,3)}') repo=${4:-"http://packages.shapeblue.com/cloudstack/upstream/centos7/4.15"} - storage=${5:-"/export/testing/4.15"} + uuid=$(cat /proc/sys/kernel/random/uuid | sed 's/-.*//g') env="qa$id-$name-$uuid-$hypervisor" echo -e "Building env with name: \033[4m$env\033[0m" echo "Management server template: $mst" echo "Hypervisor template: $hyt" - echo "Repository: $repo" - echo "Storage source: $storage" + echo "Package repo: $repo" mkdir -p $ROOT/boxes/$env - # Save env name echo $env >> $ROOT/boxes/boxes.list # Clone mgmt server @@ -230,14 +251,12 @@ deploy() { echo "Starting VMs" for domain in $(cat $ROOT/boxes/$env/list); do virsh start $domain; done; - echo "Copying primary and secondary storages" - env_storage="/export/testing/$env/" - sudo cp -vr $storage $env_storage + echo "Creating primary and secondary storage folders" + sudo mkdir -p /export/testing/$env/{primary1,primary2,secondary} sync - echo "Primary and secondary storages are ready" - echo "Initialising environment hosts" - sleep 30 + echo "Waiting for hosts to boot... (this can take a few minutes)" + sleep 60 for box in $(cat $ROOT/boxes/$env/list); do echo "Configuring host $box" ip=$(getent hosts $box | awk '{ print $1 }') @@ -337,6 +356,7 @@ deploy() { if [[ $hypervisor == "xcp" ]]; then hypervisor="xen" fi + echo $hypervisor > $ROOT/boxes/$env/hypervisor # Find VC IP export vcip= @@ -368,7 +388,6 @@ deploy() { } launch() { - set -x env=$1 if [ -z $env ]; then echo "Please provide a valid env name, run 'mbx list' to find name of environments" @@ -382,7 +401,7 @@ launch() { iscp $ROOT/boxes/$env/marvin.cfg root@$msip:/marvin/ iscp $ROOT/files/smoketests.sh root@$msip:/marvin/ iscp $ROOT/files/result.py root@$msip:/marvin/ - while ! nc -vzw 5 $msip 8096 2>&1 > /dev/null; do echo "Waiting for management server to come up"; sleep 10; done + while ! nc -vzw 5 $msip 8096 2>&1 > /dev/null; do echo "Waiting for management server to come up..."; sleep 10; done issh root@$msip systemctl start cloudstack-usage issh root@$msip "mysql -u root --execute=\"UPDATE cloud.service_offering set ram_size=512 where vm_type='consoleproxy';\"" echo "Management server is UP now, launching data center now" @@ -392,6 +411,13 @@ launch() { issh root@$msip "pip3 install /usr/share/cloudstack-marvin/Marvin*.tar.gz" issh root@$msip "python3 /usr/local/lib/python3.6/site-packages/marvin/deployDataCenter.py -i /marvin/marvin.cfg" else + if echo "4.9 4.11 4.12 4.13 4.14 4.15" | grep -w $version > /dev/null; then + hypervisor=$(cat $ROOT/boxes/$env/hypervisor) + seed_systemvmtemplate $version $hypervisor /export/testing/$env/secondary + else + echo "Unsupported ACS version $version, please manually seed the systemvmtemplate and deploy the zone" + exit 1 + fi issh root@$msip "pip2 install /usr/share/cloudstack-marvin/Marvin*.tar.gz" issh root@$msip "python /usr/lib/python2.7/site-packages/marvin/deployDataCenter.py -i /marvin/marvin.cfg" fi @@ -410,7 +436,7 @@ test() { dev() { if [[ "$1" == "-h" ]]; then - echo "Usage: mbx dev " + echo "Usage: mbx dev " exit 0 fi @@ -433,12 +459,10 @@ dev() { check_mbxt hypervisor $hyt hypervisor=$(echo $hyt | awk '{print substr($0,6,3)}') - storage=${3:-"/export/testing/4.15"} env="dev$id-$name-$hypervisor" echo -e "Building dev env with name: \033[4m$env\033[0m" echo "Hypervisor template: $hyt" - echo "Storage source: $storage" mkdir -p $ROOT/boxes/$env # Save env name @@ -455,13 +479,11 @@ dev() { echo "Starting VMs" for domain in $(cat $ROOT/boxes/$env/list); do virsh start $domain; done; - echo "Copying primary and secondary storages" - env_storage="/export/testing/$env/" - sudo cp -vr $storage $env_storage + echo "Creating primary and secondary storage folders" + sudo mkdir -p /export/testing/$env/{primary1,primary2,secondary} sync - echo "Primary and secondary storages are ready" - echo "Initialising dev hosts" + echo "Waiting for dev host to boot" sleep 60 for box in $(cat $ROOT/boxes/$env/list); do echo "Configuring host $box" @@ -638,9 +660,9 @@ destroy() { virsh destroy $vm 2> /dev/null || true; virsh undefine $vm; done - sync sed -i "/$env/d" $ROOT/boxes/boxes.list rm -fr $ROOT/boxes/$env + sync sudo rm -fr /export/testing/$env } diff --git a/templates/cache/.gitkeep b/templates/cache/.gitkeep new file mode 100644 index 0000000..e69de29