[Mobile] Add BrowserStack Android MAUI Test

csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json

@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "browserstack-sdk": {
+      "version": "1.16.13",
+      "commands": [
+        "browserstack-sdk"
+      ],
+      "rollForward": false
+    }
+  }
+}

csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs

@@ -0,0 +1,74 @@
+using Newtonsoft.Json;
+using NUnit.Framework.Interfaces;
+using NUnit.Framework;
+using OpenQA.Selenium;
+using OpenQA.Selenium.Appium;
+using OpenQA.Selenium.Appium.Android;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Microsoft.ML.OnnxRuntime.Tests.Android
+{
+    public class BrowserStackTest
+    {
+        public AndroidDriver driver;
+        public BrowserStackTest() { }
+
+        [SetUp]
+        public void Init()
+        {
+            var androidOptions = new AppiumOptions
+            {
+                AutomationName = "UIAutomator2",
+                PlatformName = "Android",
+            };
+
+            driver = new AndroidDriver(new Uri("http://127.0.0.1:4723/wd/hub"), androidOptions);
+        }
+
+        /// <summary>
+        /// Sends a log to BrowserStack that is visible in the text logs and labelled as ANNOTATION.
+        /// </summary>
+        /// <param name="text">Log text to send.</param>
+        /// <param name="logLevel">Log level -- choose between info, debug, warning, and error</param>
+        public void browserStackLog(String text, String logLevel = "info")
+        {
+            String jsonToSend = String.Format("browserstack_executor: {\"action\": \"annotate\", \"arguments\": {\"data\": {0}, \"level\": {1}}}", JsonConvert.ToString(text), JsonConvert.ToString(logLevel));
+
+            ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSend);
+        }
+
+        /// <summary>
+        /// Passes the correct test status to BrowserStack and ensures the driver quits.
+        /// </summary>
+        [TearDown]
+        public void Dispose()
+        {
+            try
+            {
+                // According to https://www.browserstack.com/docs/app-automate/appium/set-up-tests/mark-tests-as-pass-fail
+                // BrowserStack doesn't know whether test assertions have passed or failed. Below handles
+                // passing the test status to BrowserStack along with any relevant information.
+                if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed)
+                {
+                    String failureMessage = TestContext.CurrentContext.Result.Message;
+                    String jsonToSendFailure = String.Format("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"failed\", \"reason\": {0}}}", JsonConvert.ToString(failureMessage));
+
+                    ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSendFailure);
+                }
+                else
+                {
+                    ((IJavaScriptExecutor)driver).ExecuteScript("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"passed\", \"reason\": \"\"}}");
+                }
+            }
+            finally
+            {
+                // will run even if exception is thrown by previous block
+                ((AndroidDriver)driver).Quit();
+            }
+        }
+    }
+}

csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+	<PropertyGroup>
+		<TargetFramework>net8.0</TargetFramework>
+		<ImplicitUsings>enable</ImplicitUsings>
+		<Nullable>enable</Nullable>
+
+		<IsPackable>false</IsPackable>
+		<IsTestProject>true</IsTestProject>
+	</PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="Appium.WebDriver" Version="5.0.0-rc.5" />
+		<PackageReference Include="BrowserStack.TestAdapter" Version="0.13.13" />
+		<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
+		<PackageReference Include="NUnit" Version="3.13.0" />
+		<PackageReference Include="NUnit.Analyzers" Version="3.3.0" />
+		<PackageReference Include="NUnit3TestAdapter" Version="4.3.0" />
+	</ItemGroup>
+
+</Project>

csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs

@@ -0,0 +1,107 @@
+using OpenQA.Selenium.Appium;
+using OpenQA.Selenium;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Microsoft.ML.OnnxRuntime.Tests.Android
+{
+    [TestFixture]
+    public class RunAllTest : BrowserStackTest
+    {
+        public AppiumElement FindAppiumElement(String xpathQuery, String text)
+        {
+            IReadOnlyCollection<AppiumElement> appiumElements = driver.FindElements(By.XPath(xpathQuery));
+
+            foreach (var element in appiumElements)
+            {
+                if (element.Text.Contains(text))
+                {
+                    return element;
+                }
+            }
+            // was unable to find given element
+            throw new Exception(String.Format("Could not find {0}: {1} on the page.", xpathQuery, text));
+        }
+
+        public AppiumElement FindAppiumElementThenClick(String xpathQuery, String text)
+        {
+            AppiumElement appiumElement = FindAppiumElement(xpathQuery, text);
+            appiumElement.Click();
+            return appiumElement;
+        }
+
+        public (int, int) GetPassFailCount()
+        {
+            int numPassed = -1;
+            int numFailed = -1;
+
+            IReadOnlyCollection<AppiumElement> labelElements = driver.FindElements(By.XPath("//android.widget.TextView"));
+
+            for (int i = 0; i < labelElements.Count; i++)
+            {
+                AppiumElement element = labelElements.ElementAt(i);
+
+                if (element.Text.Equals("✔"))
+                {
+                    i++;
+                    numPassed = int.Parse(labelElements.ElementAt(i).Text);
+                }
+
+                if (element.Text.Equals("⛔"))
+                {
+                    i++;
+                    numFailed = int.Parse(labelElements.ElementAt(i).Text);
+                    break;
+                }
+            }
+
+            Assert.That(numPassed, Is.GreaterThanOrEqualTo(0), "Could not find number passed label.");
+            Assert.That(numFailed, Is.GreaterThanOrEqualTo(0), "Could not find number failed label.");
+
+            return (numPassed, numFailed);
+        }
+
+        [Test]
+        public async Task ClickRunAllTest()
+        {
+            AppiumElement runAllButton = FindAppiumElementThenClick("//android.widget.Button", "Run All");
+
+            while (!runAllButton.Enabled)
+            {
+                // waiting for unit tests to execute
+                await Task.Delay(500);
+            }
+
+            var (numPassed, numFailed) = GetPassFailCount();
+
+            if (numFailed == 0)
+            {
+                Assert.Pass();
+                return;
+            }
+
+            // click into test results if tests have failed
+            FindAppiumElementThenClick("//android.widget.TextView", "⛔");
+            await Task.Delay(500);
+            FindAppiumElementThenClick("//android.widget.EditText", "All");
+            await Task.Delay(100);
+            FindAppiumElementThenClick("//android.widget.TextView", "Failed");
+            await Task.Delay(500);
+
+            StringBuilder sb = new StringBuilder();
+            sb.AppendLine("PASSED TESTS: " + numPassed + " | FAILED TESTS: " + numFailed);
+
+            IReadOnlyCollection<AppiumElement> textResults = driver.FindElements(By.XPath("//android.widget.TextView"));
+            foreach (var element in textResults)
+            {
+                sb.AppendLine(element.Text);
+            }
+
+            Assert.That(numFailed, Is.EqualTo(0), sb.ToString());
+        }
+    }
+}

csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml

@@ -0,0 +1,13 @@
+app: ..\Microsoft.ML.OnnxRuntime.Tests.MAUI\bin\Release\net8.0-android\publish\ORT.CSharp.Tests.MAUI-Signed.apk
+platforms:
+  - platformName: android
+    deviceName: Samsung Galaxy S22 Ultra
+    platformVersion: 12.0
+browserstackLocal: true
+buildName: ORT android test
+buildIdentifier: ${BUILD_NUMBER}
+projectName: ORT-UITests
+debug: true
+networkLogs: false
+testContextOptions:
+    skipSessionStatus: true   

csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj

@@ -7,7 +7,7 @@
 
     <!-- General app properties -->
     <PropertyGroup>
-        <TargetFrameworks>net8.0-android;net8.0-ios;net8.0-maccatalyst</TargetFrameworks>
+        <TargetFrameworks>net8.0;net8.0-android;net8.0-ios;net8.0-maccatalyst</TargetFrameworks>
         <TargetFrameworks Condition="$([MSBuild]::IsOSPlatform('windows'))">$(TargetFrameworks);net8.0-windows10.0.19041.0</TargetFrameworks>
 
         <!-- Note for MacCatalyst:
@@ -17,7 +17,7 @@
         either BOTH runtimes must be indicated or ONLY macatalyst-x64. -->
         <!-- For example: <RuntimeIdentifiers>maccatalyst-x64;maccatalyst-arm64</RuntimeIdentifiers> -->
 
-        <OutputType>Exe</OutputType>
+        <OutputType Condition="'$(TargetFramework)' != 'net8.0'">Exe</OutputType>
         <RootNamespace>Microsoft.ML.OnnxRuntime.Tests.MAUI</RootNamespace>
         <UseMaui>true</UseMaui>
         <SingleProject>true</SingleProject>
@@ -101,6 +101,7 @@
         <PackageReference Include="DeviceRunners.VisualRunners.Xunit" Version="0.1.0-preview.2" />
         <PackageReference Include="DeviceRunners.XHarness.Maui" Version="0.1.0-preview.2" />
         <PackageReference Include="DeviceRunners.XHarness.Xunit" Version="0.1.0-preview.2" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
         <PackageReference Include="Microsoft.DotNet.XHarness.TestRunners.Xunit" Version="9.0.0-prerelease.24374.1" />
         <PackageReference Include="Microsoft.Maui.Controls" Version="8.0.70" />
         <PackageReference Include="Microsoft.Maui.Controls.Compatibility" Version="8.0.70" />
@@ -122,4 +123,10 @@
             <PackagingOutputs Remove="@(_VisualStudioTestRunnerFiles)" />
         </ItemGroup>
     </Target>
+
+	<PropertyGroup Condition="'$(IsAndroidTarget)' !='true'">
+		<GenerateProgramFile>false</GenerateProgramFile>
+		<DefaultLanguage>en</DefaultLanguage>
+	</PropertyGroup>
+
 </Project>