User/ujsrivastava/block unblock device test spec

center/src/main/java/com/microsoft/hydralab/center/service/DeviceAgentManagementService.java

@@ -29,6 +29,7 @@
 import com.microsoft.hydralab.common.entity.common.TestRun;
 import com.microsoft.hydralab.common.entity.common.TestTask;
 import com.microsoft.hydralab.common.entity.common.TestTaskSpec;
+import com.microsoft.hydralab.common.entity.common.BlockedDeviceInfo;
 import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import com.microsoft.hydralab.common.management.device.DeviceType;
 import com.microsoft.hydralab.common.repository.StatisticDataRepository;
@@ -58,6 +59,8 @@
 import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -67,8 +70,10 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.Iterator;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
@@ -81,7 +86,7 @@ public class DeviceAgentManagementService {
      * Connected session count
      */
     private final AtomicInteger onlineCount = new AtomicInteger(0);
-
+    private static volatile AtomicBoolean isUnblockingDevices = new AtomicBoolean(false);
     //save agent session <sessionId,session&agentUser>
     private final ConcurrentHashMap<String, AgentSessionInfo> agentSessionMap = new ConcurrentHashMap<>();
     //save agent info <agentId,agentInfo>
@@ -94,6 +99,9 @@ public class DeviceAgentManagementService {
     private final ConcurrentHashMap<String, AccessInfo> accessInfoMap = new ConcurrentHashMap<>();
     //save agent update info <agentId,updateTask>
     private final ConcurrentHashMap<String, AgentUpdateTask> agentUpdateMap = new ConcurrentHashMap<>();
+    // save blocked devices <deviceSerial, blockedDeviceInfo>
+    private final ConcurrentHashMap<String, BlockedDeviceInfo> blockedDevicesMap = new ConcurrentHashMap<>();
+
     @Resource
     MetricUtil metricUtil;
     @Resource
@@ -695,7 +703,7 @@ public List<AgentDeviceGroup> getAllAppiumAgents() {
 
     public JSONObject runTestTaskBySpec(TestTaskSpec testTaskSpec) {
         JSONObject result;
-
+        unblockFrozenBlockedDevices();
         if (Task.RunnerType.APPIUM_CROSS.name().equals(testTaskSpec.runningType)) {
             result = runAppiumTestTask(testTaskSpec);
         } else if (Task.RunnerType.T2C_JSON.name().equals(testTaskSpec.runningType)) {
@@ -921,12 +929,21 @@ private JSONObject runTestTaskByGroup(TestTaskSpec testTaskSpec) {
                 Assert.isTrue(!isAll, "Device/Agent Offline!");
                 continue;
             }
+
+            if (isDeviceBlocked(deviceSerial)) {
+                Assert.isTrue(!isAll, "Some of the devices in the device group are blocked!");
+                continue;
+            }
+
             isAllOffline = false;
             if (device.isOnline()) {
                 List<String> devices = testAgentDevicesMap.getOrDefault(device.getAgentId(), new ArrayList<>());
                 devices.add(device.getSerialNum());
                 testAgentDevicesMap.put(device.getAgentId(), devices);
                 testTaskSpec.agentIds.add(device.getAgentId());
+                if (testTaskSpec.blockDevice) {
+                    blockDevice(deviceSerial, testTaskSpec.testTaskId);
+                }
                 if (isSingle) {
                     break;
                 }
@@ -949,6 +966,10 @@ private JSONObject runTestTaskByGroup(TestTaskSpec testTaskSpec) {
                 result.put(Const.Param.TEST_DEVICE_SN, result.get(Const.Param.TEST_DEVICE_SN) + "," + groupDevices);
             }
             testTaskSpec.groupDevices = groupDevices;
+            if (testTaskSpec.blockDevice) {
+                testTaskSpec.blockedDeviceSerialNumber = groupDevices;
+                testTaskSpec.unblockDeviceSecretKey = testTaskSpec.testTaskId;
+            }
             message.setBody(testTaskSpec);
             sendMessageToSession(agentSessionInfoByAgentId.session, message);
         }
@@ -957,23 +978,34 @@ private JSONObject runTestTaskByGroup(TestTaskSpec testTaskSpec) {
     }
 
     private JSONObject runTestTaskByDevice(TestTaskSpec testTaskSpec) {
+        if (testTaskSpec.unblockDevice) {
+            unBlockDevice(testTaskSpec.deviceIdentifier, testTaskSpec.unblockDeviceSecretKey);
+            testTaskSpec.unblockedDeviceSerialNumber = testTaskSpec.deviceIdentifier;
+        }
+
         JSONObject result = new JSONObject();
 
         DeviceInfo device = deviceListMap.get(testTaskSpec.deviceIdentifier);
         Assert.notNull(device, "error deviceIdentifier!");
         Message message = new Message();
         message.setBody(testTaskSpec);
         message.setPath(Const.Path.TEST_TASK_RUN);
+
         Assert.isTrue(device.isAlive(), "Device/Agent Offline!");
-        if (device.isTesting()) {
+        if (device.isTesting() || isDeviceBlocked(testTaskSpec.deviceIdentifier)) {
             return result;
         }
         AgentSessionInfo agentSessionInfoByAgentId = getAgentSessionInfoByAgentId(device.getAgentId());
+
         Assert.notNull(agentSessionInfoByAgentId, "Device/Agent Offline!");
+
         if (isAgentUpdating(agentSessionInfoByAgentId.agentUser.getId())) {
             return result;
         }
         updateDeviceStatus(device.getSerialNum(), DeviceInfo.TESTING, testTaskSpec.testTaskId);
+        if (testTaskSpec.blockDevice) {
+            blockDevice(testTaskSpec.deviceIdentifier, testTaskSpec.testTaskId);
+        }
         testTaskSpec.agentIds.add(device.getAgentId());
         sendMessageToSession(agentSessionInfoByAgentId.session, message);
         result.put(Const.Param.TEST_DEVICE_SN, testTaskSpec.deviceIdentifier);
@@ -1108,6 +1140,79 @@ public int getAliveDeviceNum() {
         return (int) deviceListMap.values().stream().filter(DeviceInfo::isAlive).count();
     }
 
+    public void blockDevice(String deviceIdentifier, String testTaskId) {
+        synchronized (blockedDevicesMap) {
+            if (blockedDevicesMap.containsKey(deviceIdentifier)) {
+                throw new IllegalArgumentException("Device is already blocked by some other task!");
+            }
+
+            BlockedDeviceInfo blockedDeviceInfo = new BlockedDeviceInfo();
+            blockedDeviceInfo.setBlockedTime(Instant.now());
+            blockedDeviceInfo.setBlockingTaskUUID(testTaskId);
+            blockedDeviceInfo.setBlockedDeviceSerialNumber(deviceIdentifier);
+            blockedDevicesMap.put(deviceIdentifier,blockedDeviceInfo);
+        }
+    }
+
+    public boolean isDeviceBlocked(String deviceIdentifier) {
+        synchronized (blockedDevicesMap) {
+            return blockedDevicesMap.containsKey(deviceIdentifier);
+        }
+    }
+
+    public boolean areAllDevicesBlocked(String deviceIdentifier) {
+        if (deviceIdentifier.startsWith(Const.DeviceGroup.GROUP_NAME_PREFIX)) {
+            Set<String> deviceSerials = queryDeviceByGroup(deviceIdentifier);
+            synchronized (blockedDevicesMap) {
+                for (String deviceSerial : deviceSerials) {
+                    if (!blockedDevicesMap.containsKey(deviceSerial)) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public void unBlockDevice(String deviceIdentifier, String unblockDeviceSecretKey) {
+        synchronized (blockedDevicesMap) {
+            if (blockedDevicesMap.containsKey(deviceIdentifier)) {
+                BlockedDeviceInfo blockedDeviceInfo = blockedDevicesMap.get(deviceIdentifier);
+                if (blockedDeviceInfo.getBlockingTaskUUID().equals(unblockDeviceSecretKey)) {
+                    blockedDevicesMap.remove(deviceIdentifier);
+                } else {
+                    throw new IllegalArgumentException("Invalid unblock device secret key!");
+                }
+            } else {
+                log.info("Device {} is already unblocked.", deviceIdentifier);
+            }
+        }
+    }
+
+    public void unblockFrozenBlockedDevices() {
+
+        if (isUnblockingDevices.get()){
+            return;
+        }
+        isUnblockingDevices.set(true);
+        synchronized (blockedDevicesMap) {
+            Instant currentTime = Instant.now();
+            Iterator<Map.Entry<String, BlockedDeviceInfo>> iterator = blockedDevicesMap.entrySet().iterator();
+            while (iterator.hasNext()) {
+                Map.Entry<String, BlockedDeviceInfo> entry = iterator.next();
+                Instant blockedTime = entry.getValue().getBlockedTime();
+                Duration durationBlocked = Duration.between(blockedTime, currentTime);
+                if (durationBlocked.compareTo(Const.DeviceGroup.BLOCKED_DEVICE_TIMEOUT) > 0) {
+                    log.info("Unblocking device {} since it has been blocked for more than {} hours.", entry.getKey(), durationBlocked);
+                    iterator.remove();
+                }
+            }
+        }
+        isUnblockingDevices.set(false);
+    }
+
     @Scheduled(cron = "0 * * * * *")
     public void recordStatisticData() {
         int currentAgentNum = getAliveAgentNum();

center/src/main/java/com/microsoft/hydralab/center/service/TestTaskService.java

@@ -66,8 +66,16 @@ public Boolean isDeviceFree(String deviceIdentifier) {
                 relatedIdentifiers.addAll(deviceAgentManagementService.queryGroupByDevice(deviceIdentifier));
             }
         } else if (deviceIdentifier.startsWith(Const.DeviceGroup.GROUP_NAME_PREFIX)) {
+            if (deviceAgentManagementService.areAllDevicesBlocked(deviceIdentifier)) {
+                logger.warn("All Devices in the DeviceGroup " + deviceIdentifier + " are blocked currently.");
+                return false;
+            }
             relatedIdentifiers.addAll(deviceAgentManagementService.queryDeviceByGroup(deviceIdentifier));
         } else {
+            if (deviceAgentManagementService.isDeviceBlocked(deviceIdentifier)) {
+                logger.warn("Device " + deviceIdentifier + " is blocked currently.");
+                return false;
+            }
             relatedIdentifiers.addAll(deviceAgentManagementService.queryGroupByDevice(deviceIdentifier));
         }
         synchronized (taskQueue) {
@@ -92,6 +100,7 @@ public void runTask() {
             return;
         }
         isRunning.set(true);
+
         synchronized (taskQueue) {
             Iterator<TestTaskSpec> queueIterator = taskQueue.iterator();
             while (queueIterator.hasNext()) {

common/src/main/java/com/microsoft/hydralab/common/entity/common/BlockedDeviceInfo.java

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.hydralab.common.entity.common;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.time.Instant;
+
+@Getter
+@Setter
+@ToString
+public class BlockedDeviceInfo {
+    public Instant blockedTime;
+    public String blockingTaskUUID;
+    public String blockedDeviceSerialNumber;
+}

common/src/main/java/com/microsoft/hydralab/common/entity/common/TestTaskSpec.java

@@ -57,6 +57,11 @@ public class TestTaskSpec {
     public boolean enableNetworkMonitor;
     public String networkMonitorRule;
     public boolean enableTestOrchestrator = false;
+    public boolean blockDevice = false;
+    public boolean unblockDevice = false;
+    public String blockedDeviceSerialNumber;
+    public String unblockedDeviceSerialNumber;
+    public String unblockDeviceSecretKey;
 
     public void updateWithDefaultValues() {
         determineScopeOfTestCase();

common/src/main/java/com/microsoft/hydralab/common/util/Const.java

@@ -3,6 +3,7 @@
 
 package com.microsoft.hydralab.common.util;
 
+import java.time.Duration;
 import java.util.List;
 
 public interface Const {
@@ -32,6 +33,7 @@ interface DeviceGroup {
         String SINGLE_TYPE = "SINGLE";
         String REST_TYPE = "REST";
         String ALL_TYPE = "ALL";
+        Duration BLOCKED_DEVICE_TIMEOUT = Duration.ofHours(4);
     }
 
     interface AgentConfig {

gradle_plugin/src/main/groovy/com/microsoft/hydralab/ClientUtilsPlugin.groovy

@@ -11,6 +11,7 @@ import com.microsoft.hydralab.utils.YamlParser
 import org.apache.commons.lang3.StringUtils
 import org.gradle.api.Plugin
 import org.gradle.api.Project
+import org.gradle.internal.impldep.com.sun.xml.bind.v2.runtime.reflect.opt.Const
 
 
 class ClientUtilsPlugin implements Plugin<Project> {
@@ -173,6 +174,18 @@ class ClientUtilsPlugin implements Plugin<Project> {
                     // add quotes back as quotes in gradle plugins will be replaced by blanks
                     testConfig.analysisConfigsStr = project.analysisConfigsStr.replace("\\", "\"")
                 }
+                if (project.hasProperty('blockDevice')) {
+                    // block a device from a group of devices
+                    testConfig.blockDevice = project.blockDevice
+                }
+                if (project.hasProperty('unblockDevice')) {
+                   // unblock a device
+                    testConfig.unblockDevice = project.unblockDevice
+                }
+                if (project.hasProperty('unblockDeviceSecretKey')) {
+                    // secret key to unblock a device
+                    testConfig.unblockDeviceSecretKey = project.unblockDeviceSecretKey
+                }
 
                 requiredParamCheck(apiConfig, testConfig)
 
@@ -211,6 +224,15 @@ class ClientUtilsPlugin implements Plugin<Project> {
                 if (StringUtils.isBlank(testConfig.testSuiteName)) {
                     throw new IllegalArgumentException('Running type ' + testConfig.runningType + ' required param testSuiteName not provided!')
                 }
+                if (testConfig.unblockDevice && StringUtils.isBlank(testConfig.unblockDeviceSecretKey)) {
+                    throw new IllegalArgumentException('Running type ' + testConfig.runningType + ' required param unblockDeviceSecretKey not provided!')
+                }
+                if (testConfig.blockDevice && testConfig.unblockDevice) {
+                    throw new IllegalArgumentException('Running type ' + testConfig.runningType + ' param block and unblock device should not be true in the same test task!')
+                }
+                if(testConfig.unblockDevice && testConfig.deviceConfig.deviceIdentifier.startsWith("G.")) {
+                    throw new IllegalArgumentException('Running type ' + testConfig.runningType + ' param deviceIdentifier should not be a Group when unblockDevice is set to true!')
+                }
                 break
             case "APPIUM":
                 if (StringUtils.isBlank(testConfig.testAppPath)) {

gradle_plugin/src/main/groovy/com/microsoft/hydralab/config/TestConfig.java

@@ -60,6 +60,9 @@ public class TestConfig {
     public boolean enableTestOrchestrator = false;
     public List<AnalysisConfig> analysisConfigs = new ArrayList<>();
     public String analysisConfigsStr = "";
+    public Boolean blockDevice = false;
+    public Boolean unblockDevice = false;
+    public String unblockDeviceSecretKey = "";
 
     public void constructField(HashMap<String, Object> map) {
         Object queueTimeOutSeconds = map.get("queueTimeOutSeconds");

gradle_plugin/src/main/groovy/com/microsoft/hydralab/entity/TestTask.java

@@ -19,6 +19,9 @@ public class TestTask {
     public String testErrorMsg;
     public String message;
     public int retryTime;
+    public String blockedDeviceSerialNumber;
+    public String unblockedDeviceSerialNumber;
+    public String unblockDeviceSecretKey;
 
     @Override
     public String toString() {

gradle_plugin/src/main/groovy/com/microsoft/hydralab/utils/HydraLabAPIClient.java

@@ -240,6 +240,9 @@ public JsonObject triggerTestRun(TestConfig testConfig, HydraLabAPIConfig apiCon
         jsonElement.addProperty("networkMonitorRule", testConfig.networkMonitorRule);
         jsonElement.addProperty("enableTestOrchestrator", testConfig.enableTestOrchestrator);
         jsonElement.addProperty("notifyUrl", testConfig.notifyUrl);
+        jsonElement.addProperty("blockDevice", testConfig.blockDevice);
+        jsonElement.addProperty("unblockDevice", testConfig.unblockDevice);
+        jsonElement.addProperty("unblockDeviceSecretKey", testConfig.unblockDeviceSecretKey);
 
         try {
             if (testConfig.neededPermissions.size() > 0) {

gradle_plugin/src/main/groovy/com/microsoft/hydralab/utils/HydraLabClientUtils.java

@@ -224,6 +224,15 @@ private static void runTestInner(String reportFolderPath, HydraLabAPIConfig apiC
 
         assertNotNull(runningTest, "runningTest");
         assertNotNull(runningTest.deviceTestResults, "runningTest.deviceTestResults");
+        if (testConfig.blockDevice) {
+            assertNotNull(runningTest.blockedDeviceSerialNumber, "blockedDeviceSerialNumber");
+            printlnf("##vso[task.setvariable variable=BlockedDeviceSerialNumber;isOutput=true]%s", runningTest.blockedDeviceSerialNumber);
+            printlnf("##vso[task.setvariable variable=UnblockDeviceSecretKey;isOutput=true]%s", runningTest.unblockDeviceSecretKey);
+        }
+
+        if (testConfig.unblockDevice && testConfig.deviceConfig.deviceIdentifier.equals(runningTest.unblockedDeviceSerialNumber)) {
+            printlnf("##[section] Device % s, unblocked.", runningTest.unblockedDeviceSerialNumber);
+        }
 
         String testReportUrl = apiConfig.getTestReportUrl(runningTest.id);