Feature/upgrade to ansible-core 2.15 vagrant and vagrant-plugin (#7855)

* Update ansible requirements

* Fix ansible_default_ipv4 gathering fact

* Update Cumulus Version

* Change RHEL8 version and Downgrade Cumulus VX to still have hostapd

* Remove the modeltype for nodes 01 02 03 ; Reduce memory for cumulus

* Fix cumulus user password change

* Increase halt timeout for VM

* Change timeout for halt

* Add new runner tag to test v7

* Move to tag shell-v7

* Remove Windows timeout

* Move cumulus variables

* Packer: Add missing collections

* Add community ref

* Add ansible environement for roles and collections

* Add ansible module collection-community

* Revert "Add community ref"

This reverts commit 3f11812.

* Revert "Move to tag shell-v7"

This reverts commit 0c8afce.

* Fix by selecting only defined ipv4

* Increase debops version

* fix issue with mgmt_interface during venom variable installation

* Change check_free_space to fix #7839

* Add v7 to runner docker tag

* Revert "fix issue with mgmt_interface during venom variable installation"
It is not the right way to fix it

This reverts commit 0d5852b.

* Update the way to have interface id

* Fix #7844

* Revert "Add v7 to runner docker tag"

IT is too early, runners are not ready

This reverts commit 689fcf4.

* Add missing collection for appserver

* Upgrade nodejs version requierments for tests

* Add back shell-sourceforge tag for admin runners

.gitlab-ci.yml

@@ -436,7 +436,7 @@ variables:
     paths:
       - results/*
   tags:
-    - test-shell
+    - test-shell-v7
 
 .deploy_deb_job:
   stage: deploy

addons/vagrant/Vagrantfile

@@ -11,14 +11,15 @@ Vagrant.configure("2") do |config|
   # disable synced folders
   config.vm.synced_folder ".", "/vagrant", disabled: true
 
-  # The maximum amount of time to wait for a response
-  config.winrm.timeout = 600
-
   # use same private key on all machines
   config.ssh.insert_key = false
 
   # Number of seconds to wait for establishing an SSH connection to the guest
-  config.ssh.connect_timeout = 60
+  config.ssh.connect_timeout = 90
+
+  # The time in seconds that Vagrant will wait for the machine to gracefully halt when vagrant halt is called.
+  # https://developer.hashicorp.com/vagrant/docs/vagrantfile/machine_settings#config-vm-graceful_halt_timeout
+  config.vm.graceful_halt_timeout = 120
 
   config.vm.provider "libvirt" do |vm|
     vm.default_prefix = DOMAIN_PREFIX

addons/vagrant/cumulus/Vagrantfile

@@ -26,6 +26,11 @@ function vagrant_user_nclu(){
     adduser vagrant netshow
 }
 
+function vagrant_user_nvue(){
+    echo "### Giving Vagrant User Ability to Run NVUE Commands ###"
+    adduser vagrant nvapply
+}
+
 if grep -q -i 'cumulus' /etc/lsb-release &> /dev/null; then
     echo "### RUNNING CUMULUS EXTRA CONFIG ###"
     source /etc/lsb-release
@@ -76,6 +81,11 @@ if grep -q -i 'cumulus' /etc/lsb-release &> /dev/null; then
             disable_remap
             vagrant_user_nclu
             setup_ztp
+        elif [[ $DISTRIB_RELEASE =~ ^5.* ]]; then
+            echo "  INFO: Detected a 5.x Based Release ($DISTRIB_RELEASE)"
+            disable_remap
+            vagrant_user_nvue
+            setup_ztp
         fi
     fi
 fi
@@ -109,12 +119,20 @@ node03_netmask = user_vars['vms']['node03']['netmask']
 
 Vagrant.configure("2") do |config|
 
+  # The time in seconds that Vagrant will wait for the machine to gracefully halt when vagrant halt is called.
+  # https://developer.hashicorp.com/vagrant/docs/vagrantfile/machine_settings#config-vm-graceful_halt_timeout
+  config.vm.graceful_halt_timeout = 120
+
   wbid = 1
   offset = wbid * 100
 
   ##### DEFINE VM for switch01 #####
   config.vm.define "switch01" do |device|
 
+
+    # Number of seconds to wait for establishing an SSH connection to the guest
+    config.ssh.connect_timeout = 120
+
     device.vm.hostname = "switch01"
 
     device.vm.box = inventory['all']['children']['cumulus']['hosts']['switch01']['box']
@@ -324,7 +342,7 @@ vagrant_interface_rule
     device.vm.box_version = inventory['all']['children']['nodes']['hosts']['node01']['box_version']
 
     device.vm.provider :libvirt do |v|
-      v.nic_model_type = 'e1000'
+      #v.nic_model_type = 'e1000'
       v.memory = 1024
 
     end
@@ -434,7 +452,7 @@ vagrant_interface_rule
     device.vm.box_version = inventory['all']['children']['nodes']['hosts']['node02']['box_version']
 
     device.vm.provider :libvirt do |v|
-      v.nic_model_type = 'e1000'
+      #v.nic_model_type = 'e1000'
       v.memory = 1024
 
     end
@@ -531,7 +549,7 @@ vagrant_interface_rule
     device.vm.box_version = inventory['all']['children']['nodes']['hosts']['node03']['box_version']
 
     device.vm.provider :libvirt do |v|
-      v.nic_model_type = 'e1000'
+      #v.nic_model_type = 'e1000'
       v.memory = 1024
 
     end

addons/vagrant/inventory/group_vars/all/common.yml

@@ -25,4 +25,3 @@ pipeline_id: '{{ lookup("env","CI_PIPELINE_ID") | default("123456789", true) }}'
 
 # get current PacketFence minor release (X.Y)
 pf_minor_release: '{{ lookup("env", "PF_MINOR_RELEASE") | default("99.9", true) }}'
-

addons/vagrant/inventory/group_vars/clusters/venom_local_vars.yml

@@ -33,10 +33,10 @@ venom_local_vars:
     value: '{{ mailhog__api_port }}'
 
   - name: 'internet_int_ip'
-    value: "{{ ansible_default_ipv4['address'] }}"
+    value: "{{ ansible_default_ipv4.address }}"
 
   - name: 'internet_int_netmask'
-    value: "{{ ansible_default_ipv4['netmask'] }}"
+    value: "{{ ansible_default_ipv4.netmask }}"
 
   - name: 'timezone'
     value: "{{ ansible_date_time['tz'] }}"

addons/vagrant/inventory/group_vars/cumulus/switch_settings.yml

@@ -0,0 +1,3 @@
+---
+switch01_api_user: cumulus
+switch01_api_password: CumulusLinux!

addons/vagrant/inventory/group_vars/pfservers/environment.yml

@@ -12,6 +12,4 @@ inventory__group_environment:
   PSONO_CI_API_KEY_ID: '{{ lookup("env", "PSONO_CI_API_KEY_ID") }}'
   PSONO_CI_API_SECRET_KEY_HEX: '{{ lookup("env", "PSONO_CI_API_SECRET_KEY_HEX") }}'
   PATH: '/usr/local/go/bin:{{ ansible_env.PATH }}'
-  CI: '{{ True if lookup("env", "CI")
-               else False }}'
-
+  CI: "{{ 'True ' if lookup('env', 'CI') else 'False ' }}"

addons/vagrant/inventory/group_vars/pfservers/packetfence_install.yml

@@ -1,7 +1,7 @@
 ---
 # hack to get id of mgmt interface based
 # on IP assigned by vagrant (in inventory/hosts file)
-mgmt_interface_id: "{{ ansible_facts | dict2items | selectattr('value.ipv4', 'defined') | selectattr('value.ipv4.address', 'equalto', mgmt_ip) | first }}"
+mgmt_interface_id: "{{ ansible_facts | dict2items | selectattr('value.ipv4', 'defined') | selectattr('value.type', 'defined') | selectattr('value.ipv4.address', 'equalto', mgmt_ip) | first }}"
 
 packetfence_install__mgmt_interface:
   id: "{{ mgmt_interface_id['key'] }}"

addons/vagrant/inventory/group_vars/standalones/venom_local_vars.yml

@@ -50,11 +50,11 @@ venom_local_vars:
     value: '{{ mailhog__api_port }}'
 
   - name: 'internet_int_ip'
-    value: "{{ ansible_default_ipv4['address'] }}"
+    value: "{{ ansible_default_ipv4.address }}"
 
   - name: 'internet_int_netmask'
-    value: "{{ ansible_default_ipv4['netmask'] }}"
-
+    value: "{{ ansible_default_ipv4.netmask }}"
+  
   - name: 'timezone'
     value: "{{ ansible_date_time['tz'] }}"
 

addons/vagrant/inventory/hosts

@@ -11,7 +11,7 @@ all:
           mgmt_ip: "{{ users_vars[dict_name]['vms']['switch01']['ip'] }}"
           mgmt_netmask: "{{ users_vars[dict_name]['vms']['switch01']['netmask'] }}"
           box: CumulusCommunity/cumulus-vx
-          box_version: 3.7.12
+          box_version: 4.3.0
           ansible_host: "{{ mgmt_ip }}"
 
     nodes:
@@ -113,7 +113,7 @@ all:
           hosts:
             pf1el8dev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pf1el8dev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pf1el8dev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -122,7 +122,7 @@ all:
               memory: 16384
             pf2el8dev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pf2el8dev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pf2el8dev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -131,7 +131,7 @@ all:
               memory: 16384
             pf3el8dev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pf3el8dev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pf3el8dev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -167,7 +167,7 @@ all:
               memory: 16384
             pf1el8localdev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pf1el8localdev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pf1el8localdev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -176,7 +176,7 @@ all:
               memory: 16384
             pf2el8localdev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pf2el8localdev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pf2el8localdev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -185,7 +185,7 @@ all:
               memory: 16384
             pf3el8localdev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pf3el8localdev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pf3el8localdev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -224,7 +224,7 @@ all:
           hosts:
             pfel8dev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pfel8dev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pfel8dev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -242,7 +242,7 @@ all:
               memory: 16384
             el8dev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['el8dev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['el8dev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -264,7 +264,7 @@ all:
               ansible_connection: local
             pfel8localdev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pfel8localdev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pfel8localdev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -282,7 +282,7 @@ all:
               memory: 16384
             el8localdev:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['el8dev']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['el8dev']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"
@@ -300,7 +300,7 @@ all:
               memory: 16384
             pfel8stable:
               box: generic/rhel8
-              box_version: '3.6.4'
+              box_version: '4.2.16'
               mgmt_ip: "{{ users_vars[dict_name]['vms']['pfel8stable']['ip'] }}"
               mgmt_netmask: "{{ users_vars[dict_name]['vms']['pfel8stable']['netmask'] }}"
               ansible_host: "{{ mgmt_ip }}"

addons/vagrant/linux_servers/Vagrantfile

@@ -18,6 +18,11 @@ user_vars = inventory['all']['vars']['users_vars'][dict_name]
 Vagrant.configure("2") do |config|
   # loop on **all** host(s) in linux_servers group in inventory to create VM(s)
   inventory['all']['children']['linux_servers']['hosts'].each do |server,details|
+
+    # The time in seconds that Vagrant will wait for the machine to gracefully halt when vagrant halt is called.
+    # https://developer.hashicorp.com/vagrant/docs/vagrantfile/machine_settings#config-vm-graceful_halt_timeout
+    config.vm.graceful_halt_timeout = 120
+
     config.vm.define server, autostart: false do |srv|
       srv.vm.box = details['box']
       srv.vm.box_version = details['box_version']

addons/vagrant/pfservers/Vagrantfile

@@ -18,6 +18,9 @@ end
 user_vars = inventory['all']['vars']['users_vars'][dict_name]
 
 Vagrant.configure("2") do |config|
+  # Number of seconds to wait for establishing an SSH connection to the guest
+  config.ssh.connect_timeout = 90
+
   # loop on **all** host(s) in pfservers group in inventory to create VM(s)
   inventory['all']['children']['pfservers']['children'].each do |groups, hosts|
     hosts['hosts'].each do |server, details|

addons/vagrant/playbooks/cumulus.yml

@@ -14,3 +14,10 @@
   roles:
     - role: hostapd
     - role: restserver
+
+  post_tasks:
+  - name: Change {{ switch01_api_user }} password user
+    ansible.builtin.user:
+      name: "{{ switch01_api_user }}"
+      password: "{{ switch01_api_password | password_hash }}"
+      force: true

addons/vagrant/playbooks/tasks/vagrant_iptables.yml

@@ -4,7 +4,7 @@
     path: "/usr/local/pf/conf/iptables-input.conf.inc"
     block: |
       # allow SSH for vagrant management
-      -A INPUT --in-interface {{ ansible_default_ipv4['interface'] }} --jump ACCEPT
+      -A INPUT --in-interface {{ ansible_default_ipv4.interface }} --jump ACCEPT
     marker: "# {mark} ANSIBLE MANAGED BLOCK - vagrant"
 
 - name: allow external services on management interface

addons/vagrant/requirements.yml

@@ -16,13 +16,17 @@ collections:
   - name: inverse_inc.packetfence
     version: 1.2.4
   - name: debops.debops
-    version: 2.3.2
+    version: 3.0.5
   - name: inverse_inc.cumulus
     version: 1.1.1
   - name: inverse_inc.utils
     version: 1.1.2
   - name: inverse_inc.wireless
     version: 0.2.2
+  - name: ansible.netcommon
+    version: 5.1.1
+  - name: ansible.posix
+    version: 1.5.4
 
 # For collections, to test locally with Vagrant (due to --force option)
 # Ansible will create an export, not a symlink to git repository

addons/vagrant/wireless/Vagrantfile

@@ -17,6 +17,10 @@ user_vars = inventory['all']['vars']['users_vars'][dict_name]
 
 Vagrant.configure("2") do |config|
   inventory['all']['children']['wireless']['hosts'].each do |server,details|
+    # The time in seconds that Vagrant will wait for the machine to gracefully halt when vagrant halt is called.
+    # https://developer.hashicorp.com/vagrant/docs/vagrantfile/machine_settings#config-vm-graceful_halt_timeout
+    config.vm.graceful_halt_timeout = 120
+
     config.vm.define server, autostart: false do |srv|
 
       srv.vm.box = details['box']

ci/packer/cpanbuild.json

@@ -57,7 +57,7 @@
             "collections_path": "{{user `ansible_collections_path`}}",
             "inventory_directory": "{{user `prov_dir`}}/inventory",
             "extra_arguments": ["--extra-vars", "{{user `ansible_extra_vars`}}"],
-            "ansible_env_vars": ["PF_MINOR_RELEASE={{ user `pf_minor_release`}}"],
+            "ansible_env_vars": ["PF_MINOR_RELEASE={{ user `pf_minor_release`}}", "ANSIBLE_ROLES_PATH={{user `prov_dir`}}/roles", "ANSIBLE_COLLECTIONS_PATH={{user `prov_dir`}}/"],
             "groups": ["{{user `ansible_centos_group`}}","{{user `ansible_centos8_group`}}"],
             "user": "{{user `ansible_user`}}",
             "only": ["{{user `builder_prefix`}}-centos-8"]
@@ -71,7 +71,7 @@
             "collections_path": "{{user `ansible_collections_path`}}",
             "inventory_directory": "{{user `prov_dir`}}/inventory",
             "extra_arguments": ["--extra-vars", "{{user `ansible_extra_vars`}}"],
-            "ansible_env_vars": ["PF_MINOR_RELEASE={{ user `pf_minor_release`}}"],
+            "ansible_env_vars": ["PF_MINOR_RELEASE={{ user `pf_minor_release`}}", "ANSIBLE_ROLES_PATH={{user `prov_dir`}}/roles", "ANSIBLE_COLLECTIONS_PATH={{user `prov_dir`}}/"],
             "groups": ["{{user `ansible_debian_group`}}"],
             "user": "{{user `ansible_user`}}",
             "only": ["{{user `builder_prefix`}}-bullseye"]

ci/packer/pfbuild.json

@@ -63,7 +63,7 @@
             "collections_path": "{{user `ansible_collections_path`}}",
             "inventory_directory": "{{user `prov_dir`}}/inventory",
             "extra_arguments": ["--extra-vars", "{{user `ansible_extra_vars`}}"],
-            "ansible_env_vars": ["PF_MINOR_RELEASE={{ user `pf_minor_release`}}"],
+            "ansible_env_vars": ["PF_MINOR_RELEASE={{ user `pf_minor_release`}}", "ANSIBLE_ROLES_PATH={{user `prov_dir`}}/roles", "ANSIBLE_COLLECTIONS_PATH={{user `prov_dir`}}/"],
             "groups": ["{{user `ansible_centos_group`}}","{{user `ansible_centos8_group`}}"],
             "user": "{{user `ansible_user`}}",
             "only": ["{{user `builder_prefix`}}-centos-8"]
@@ -77,7 +77,7 @@
             "collections_path": "{{user `ansible_collections_path`}}",
             "inventory_directory": "{{user `prov_dir`}}/inventory",
             "extra_arguments": ["--extra-vars", "{{user `ansible_extra_vars`}}"],
-            "ansible_env_vars": ["PF_MINOR_RELEASE={{ user `pf_minor_release`}}"],
+            "ansible_env_vars": ["PF_MINOR_RELEASE={{ user `pf_minor_release`}}", "ANSIBLE_ROLES_PATH={{user `prov_dir`}}/roles", "ANSIBLE_COLLECTIONS_PATH={{user `prov_dir`}}/"],
             "groups": ["{{user `ansible_debian_group`}}"],
             "user": "{{user `ansible_user`}}",
             "only": ["{{user `builder_prefix`}}-bullseye"]

ci/packer/provisionners_cpanbuild/playbooks/python3.yml

@@ -3,6 +3,8 @@
   name: Install and configure Python 3 for rpmlint and nodejs
   become: True
   gather_facts: True
+  collections:
+    - community.general
 
   tasks:
     - name: Install Python 3 packages

ci/packer/provisionners_cpanbuild/requirements.yml

@@ -2,3 +2,6 @@
 roles:
   - src: inverse-inc.gitlab_buildpkg_tools
     name: inverse_inc.gitlab_buildpkg_tools
+collections:
+  - name: community.general
+    version: 7.0.1

ci/packer/provisionners_pfbuild/playbooks/python3.yml

@@ -3,6 +3,8 @@
   name: Install and configure Python 3 for rpmlint and nodejs
   become: True
   gather_facts: True
+  collections:
+    - community.general
 
   tasks:
     - name: Install Python 3 packages