diff --git a/build.gradle b/build.gradle index a9ae7e5..8dcd53d 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ plugins { id 'com.gradle.plugin-publish' version '0.11.0' } -version '3.1.2' +version '3.1.3' pluginBundle { website = 'https://www.browserstack.com' @@ -17,7 +17,7 @@ pluginBundle { description = 'Runs Espresso tests on BrowserStack' tags = ['espresso', 'test', 'browserstack', 'app', 'automate', 'app-automate', 'appautomate', 'app-live', 'applive'] - version = '3.1.2' + version = '3.1.3' } } } diff --git a/src/main/java/com/browserstack/gradle/BrowserStackPlugin.java b/src/main/java/com/browserstack/gradle/BrowserStackPlugin.java index e2e8d89..064694f 100644 --- a/src/main/java/com/browserstack/gradle/BrowserStackPlugin.java +++ b/src/main/java/com/browserstack/gradle/BrowserStackPlugin.java @@ -41,7 +41,7 @@ public void apply(Project project) { task.setGroup(DEFAULT_GROUP); task.setDescription("Uploads app / tests to AppAutomate and executes them"); // Run Espresso tests without building the apk and test apk - if (!project.hasProperty("skipBuildingApks")) { + if (!project.hasProperty("skipBuildingApks") || Boolean.parseBoolean(project.property("skipBuildingApks").toString()) == false) { task.dependsOn("assemble" + appVariantName, "assemble" + appVariantName + "AndroidTest"); } task.setAppVariantBaseName(applicationVariant.getBaseName()); @@ -51,6 +51,12 @@ public void apply(Project project) { task.setConfigFilePath(browserStackConfigExtension.getConfigFilePath()); task.setHost(Constants.BROWSERSTACK_API_HOST); task.setDebug(browserStackConfigExtension.isDebug()); + if(project.hasProperty("mainAPKPath")){ + task.setMainAPKPath(project.property("mainAPKPath").toString()); + } + if(project.hasProperty("testAPKPath")){ + task.setTestAPKPath(project.property("testAPKPath").toString()); + } }); project.getTasks().create("upload" + appVariantName + "ToBrowserstackAppLive", AppLiveUploadTask.class, (task) -> { diff --git a/src/main/java/com/browserstack/gradle/BrowserStackTask.java b/src/main/java/com/browserstack/gradle/BrowserStackTask.java index 02c77b5..696ad7a 100644 --- a/src/main/java/com/browserstack/gradle/BrowserStackTask.java +++ b/src/main/java/com/browserstack/gradle/BrowserStackTask.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; @@ -40,6 +41,14 @@ public class BrowserStackTask extends DefaultTask { @Optional public String command ; + @Input + @Optional + public String mainAPKPath; + + @Input + @Optional + public String testAPKPath; + public void setAppVariantBaseName(String appVariantBaseName) { this.appVariantBaseName = appVariantBaseName; } @@ -80,6 +89,14 @@ public void setHost(String host) { public void setCommand(String command) { this.command = command; } + public String getMainAPKPath() { return mainAPKPath; } + + public void setMainAPKPath(String mainAPKPath) { this.mainAPKPath = mainAPKPath; } + + public String getTestAPKPath() {return testAPKPath; } + + public void setTestAPKPath(String testAPKPath) { this.testAPKPath = testAPKPath; } + protected JSONObject constructDefaultBuildParams() { JSONObject params = new JSONObject(); params.put("app", app); @@ -152,21 +169,60 @@ public static Path findMostRecentPath(List paths) { return mostRecentPath; } + private boolean isPathRelative(String apkPath){ + if(apkPath.startsWith("./")){ + return true; + } + return false; + } + private String getAbsolutePath(String apkPath, String currentWorkingDirectory){ + if(isPathRelative(apkPath)){ + return currentWorkingDirectory + apkPath.substring(1); + } + return apkPath; + } public Map locateApks(boolean ignoreTestPath) throws IOException { Path debugApkPath; Path testApkPath; String dir = System.getProperty("user.dir"); List appApkFiles = new ArrayList<>(); List testApkFiles = new ArrayList<>(); - Files.find(Paths.get(dir), Constants.APP_SEARCH_MAX_DEPTH, (filePath, fileAttr) -> isValidFile(filePath, fileAttr)) - .forEach(f -> { - - if (f.toString().endsWith("-androidTest.apk")) { - testApkFiles.add(f); - } else { - appApkFiles.add(f); - } - }); + final Boolean[] isAPKFileCreated = {false,false}; // 1st element stores true if main apk is read from path provided by client and false otherwise. 2nd element is for test apk. + if(mainAPKPath != null){ + isAPKFileCreated[0] = true; + try { + Files.find(Paths.get(getAbsolutePath(mainAPKPath, dir)), 1, (filePath, fileAttr) -> isValidAPKFile(filePath, fileAttr)) + .forEach(f -> { + appApkFiles.add(f); + }); + }catch (NoSuchFileException e ){ + throw new IOException("Invalid File Path: Please provide a valid main APK path"); + } + } + if(testAPKPath != null){ + isAPKFileCreated[1] = true; + try { + Files.find(Paths.get(getAbsolutePath(testAPKPath, dir)), 1, (filePath, fileAttr) -> isValidAPKFile(filePath, fileAttr)) + .forEach(f -> { + testApkFiles.add(f); + }); + }catch(NoSuchFileException e ){ + throw new IOException("Invalid File Path: Please provide a valid test APK path"); + } + } + + if(!isAPKFileCreated[0] || !isAPKFileCreated[1]) { + Files.find(Paths.get(dir), Constants.APP_SEARCH_MAX_DEPTH, (filePath, fileAttr) -> isValidFile(filePath, fileAttr)) + .forEach(f -> { + if (f.toString().endsWith("-androidTest.apk")) { + if(!isAPKFileCreated[1]) { + testApkFiles.add(f); + } + } else if (!isAPKFileCreated[0]) { + appApkFiles.add(f); + } + }); + } debugApkPath = findMostRecentPath(appApkFiles); testApkPath = findMostRecentPath(testApkFiles); @@ -188,7 +244,10 @@ public Map locateApks(boolean ignoreTestPath) throws IOException { } private boolean isValidFile(Path filePath, BasicFileAttributes fileAttr) { - return fileAttr.isRegularFile() && filePath.toString().endsWith(".apk") && filePath.getFileName().toString() + return isValidAPKFile(filePath, fileAttr) && filePath.getFileName().toString() .contains(appVariantBaseName); } + private boolean isValidAPKFile(Path filePath, BasicFileAttributes fileAttr) { + return fileAttr.isRegularFile() && filePath.toString().endsWith(".apk") ; + } } diff --git a/test.rb b/test.rb index 1ab2eb7..63f2f40 100644 --- a/test.rb +++ b/test.rb @@ -9,10 +9,21 @@ def print_separator puts "\n******************************************************************************************************" end +def setup_required_paths + $current_path = $current_path.chop + $correct_absolute_mainAPKPath = $current_path + "/test/mainApk" + $incorrect_absolute_APKPath = "/random_path/" + $correct_absolute_testAPKPath = $current_path + "/test/testApk" + $correct_relative_mainAPKPath = "./test/mainApk" + $incorrect_relative_APKPath = "./random_path/" + $correct_relative_testAPKPath = "./test/testApk" +end + def setup_repo puts "Setting up sample repo" run_command("git clone https://github.com/browserstack/espresso-browserstack.git") Dir.chdir "espresso-browserstack" + $current_path = run_command("pwd"); run_command("git checkout gradlePluginTestBranch") end @@ -34,6 +45,78 @@ def run_basic_espresso_test(gradle_command) end end +def run_espresso_test_with_path(gradle_command) + puts "Running #{gradle_command} with config and path to main and test apk" + stdout = run_command(gradle_command) + responses = stdout.lines.select{ |line| line.match(/app_url|test_suite_url|build_id/)} + if responses.count != 3 + puts "✘ #{gradle_command} failed with error: #{responses}".red + else + puts "✔ #{gradle_command} tests passed".green + puts responses.join("\n") + end +end + +def run_espresso_test_with_invalid_cucumber_config(gradle_command) + puts "Running #{gradle_command} with invalid cucumber config and path to main and test apk" + stdout = run_command(gradle_command) + responses = stdout.lines.select{ |line| line.match(/app_url|test_suite_url|422|BROWSERSTACK_INVALID_VALUES_IN_INPUT/)} + if responses.count != 4 + puts "✘ #{gradle_command} failed with error: #{responses}".red + else + puts "✔ #{gradle_command} tests passed".green + puts responses.join("\n") + end +end + +def run_espresso_test_with_invalid_instrumentation_config(gradle_command) + puts "Running #{gradle_command} with invalid instrumentation config and path to main and test apk" + stdout = run_command(gradle_command) + responses = stdout.lines.select{ |line| line.match(/app_url|test_suite_url|422|contain characters/)} + if responses.count != 4 + puts "✘ #{gradle_command} failed with error: #{responses}".red + else + puts "✔ #{gradle_command} tests passed".green + puts responses.join("\n") + end +end + +def run_espresso_test_with_incorrect_path(gradle_command) + puts "Running #{gradle_command} with basic config and incorrect paths to both main and test apk" + stdout = run_command(gradle_command) + responses = stdout.lines.select{ |line| line.match(/DebugApp apk: null|TestApp apk: null/)} + if responses.count != 2 + puts "✘ #{gradle_command} failed with error: #{responses}".red + else + puts "✔ #{gradle_command} tests passed".green + puts responses.join("\n") + end +end + +def run_espresso_test_with_either_main_or_test_apk_path(gradle_command, apk_path) + puts "Running #{gradle_command} with basic config and either main apk path or test apk path" + stdout = run_command(gradle_command) + responses = stdout.lines.select{ |line| line.match(/#{apk_path}|test_suite_url|build_id/)} + if responses.count != 3 + puts "✘ #{gradle_command} failed with error: #{responses}".red + else + puts "✔ #{gradle_command} tests passed".green + puts responses.join("\n") + end +end + +def run_espresso_test_with_one_absolute_and_one_relative_apk_path(gradle_command, main_apk_path, test_apk_path) + puts "Running #{gradle_command} with basic config and one of the paths as absolute and one as relative" + stdout = run_command(gradle_command) + responses = stdout.lines.select{ |line| line.match(/#{main_apk_path}|#{test_apk_path}|test_suite_url|build_id/)} + if responses.count != 4 + puts "✘ #{gradle_command} failed with error: #{responses}".red + else + puts "✔ #{gradle_command} tests passed".green + puts responses.join("\n") + end +end + def run_app_live_test(gradle_command) puts "Running #{gradle_command}" stdout = run_command(gradle_command) @@ -59,9 +142,149 @@ def run_tests_args puts "\nRunning new tests using ./gradlew with args" run_basic_espresso_test("./gradlew clean executeDebugTestsOnBrowserstack --config-file='command-line-config-browserstack.json'") run_basic_espresso_test("./gradlew executeDebugTestsOnBrowserstack -PskipBuildingApks=true") + run_basic_espresso_test("./gradlew executeDebugTestsOnBrowserstack -PskipBuildingApks=false") print_separator end +def run_tests_with_path_args + puts "\nRunning new test using ./gradlew with APK paths for main and test apk" + mainAPKPath = $current_path + "/test/mainApk" + testAPKPAth = $current_path + "/test/testApk" + run_espresso_test_with_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth}") +end + +def run_tests_with_relative_path + puts "\nRunning new test using ./gradlew with relative paths for both main and test apk" + mainAPKPath = "./test/mainApk" + testAPKPAth = "./test/testApk" + run_espresso_test_with_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth}") +end + +def run_test_with_incorrect_path + puts "\nRunning new test using ./gradlew with incorrect main and test APK paths" + mainAPKPath = $current_path + testAPKPAth = $current_path + run_espresso_test_with_incorrect_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth}") +end + +def run_test_with_incorrect_relative_path + puts "\nRunning new test using ./gradlew with incorrect relative paths for both main and test APK " + mainAPKPath = "./test" + testAPKPAth = "./test" + run_espresso_test_with_incorrect_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth}") +end + + +def run_tests_with_path_variations + puts "\nRunning new test using ./gradlew with mainAPKPath arg only" + mainAPKPath = $current_path + "/test/mainApk" + run_espresso_test_with_either_main_or_test_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath}", mainAPKPath); + puts "\nRunning new test using ./gradlew with testAPKPath arg only" + testAPKPAth = $current_path + "/test/testApk" + run_espresso_test_with_either_main_or_test_apk_path("./gradlew executeDebugTestsOnBrowserstack -PtestAPKPath=#{testAPKPAth}", testAPKPAth); +end + +def run_tests_with_relative_path_variations + puts "\nRunning new test using ./gradlew with mainAPKPath(relative path) arg only" + mainAPKPath = "./test/mainApk" + run_espresso_test_with_either_main_or_test_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath}", mainAPKPath); + puts "\nRunning new test using ./gradlew with testAPKPath(relative path) arg only" + testAPKPAth = "./test/testApk" + run_espresso_test_with_either_main_or_test_apk_path("./gradlew executeDebugTestsOnBrowserstack -PtestAPKPath=#{testAPKPAth}", testAPKPAth); + puts "\nRunning with absolute main and relative test path" + mainAPKPath = $current_path + "/test/mainApk" + run_espresso_test_with_one_absolute_and_one_relative_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth}", mainAPKPath, testAPKPAth) + puts "\nRunning with absolute test and relative main path" + testAPKPAth = $current_path + "/test/testApk" + mainAPKPath = "./test/mainApk" + run_espresso_test_with_one_absolute_and_one_relative_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth}", mainAPKPath, testAPKPAth) +end + +def run_espresso_test_with_non_existing_apk_path(gradle_command, apk_type) + puts "Running #{gradle_command} with non existing path" + stdout = run_command(gradle_command) + responses = stdout.lines.select{ |line| line.match(/Invalid File Path: Please provide a valid #{apk_type} APK path/)} + if responses.count != 1 + puts "✘ #{gradle_command} failed with error: #{responses}".red + else + puts "✔ #{gradle_command} tests passed".green + puts responses.join("\n") + end +end + +def run_espresso_test_with_one_existing_and_one_non_existing_apk_path(gradle_command, incorrect_apk_type) + puts "Running #{gradle_command}" + stdout = run_command(gradle_command) + response_line_with_incorrect_path = stdout.lines.select{ |line| line.match(/Invalid File Path: Please provide a valid #{incorrect_apk_type} APK path/)} + response_line_with_build_id = stdout.lines.select{ |line| line.match(/build_id/)} + if response_line_with_incorrect_path.count != 1 && response_line_with_build_id != 0 + puts "✘ #{gradle_command} failed with error: #{response_line_with_incorrect_path}".red + else + puts "✔ #{gradle_command} tests passed".green + puts response_line_with_incorrect_path.join("\n") + end +end + +def run_espresso_test_with_one_incorrect_apk_path(gradle_command) + puts "Running #{gradle_command}" + stdout = run_command(gradle_command) + responses = stdout.lines.select{ |line| line.match(/TestApp apk: null/)} + if responses.count != 1 + puts "✘ #{gradle_command} failed with error: #{responses}".red + else + puts "✔ #{gradle_command} tests passed".green + puts responses.join("\n") + end +end + +def run_test_with_non_existing_path + puts "\nRunning new test with non existing absolute main path" + mainAPKPath = "/random_path/" + run_espresso_test_with_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath}", "main"); + puts "\nRunning new test with non existing relative main path" + mainAPKPath = "./random_path/" + run_espresso_test_with_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath}", "main"); + puts "\nRunning new test with non existing absolute test path" + testAPKPath = "/random_path/" + run_espresso_test_with_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PtestAPKPath=#{testAPKPath}", "test"); + puts "\nRunning new test with non existing absolute test path" + testAPKPath = "./random_path/" + run_espresso_test_with_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PtestAPKPath=#{testAPKPath}", "test"); +end + +def run_test_with_one_correct_and_one_non_existing_path + puts "\nRunning new test with absolute correct test and non existing absolute main path" + run_espresso_test_with_one_existing_and_one_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{$incorrect_absolute_APKPath} -PtestAPKPath=#{$correct_absolute_testAPKPath}", "main"); + puts "\nRunning new test with relative correct test and non existing absolute main path" + run_espresso_test_with_one_existing_and_one_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{$incorrect_absolute_APKPath} -PtestAPKPath=#{$correct_relative_testAPKPath}", "main"); + puts "\nRunning new test with absolute correct test and non existing relative main path" + run_espresso_test_with_one_existing_and_one_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{$incorrect_relative_APKPath} -PtestAPKPath=#{$correct_absolute_testAPKPath}", "main"); + puts "\nRunning new test with relative correct test and non existing relative main path" + run_espresso_test_with_one_existing_and_one_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{$incorrect_relative_APKPath} -PtestAPKPath=#{$correct_relative_testAPKPath}", "main"); + puts "\nRunning new test with absolute correct main and non existing absolute test path" + run_espresso_test_with_one_existing_and_one_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{$correct_absolute_mainAPKPath} -PtestAPKPath=#{$incorrect_absolute_APKPath}", "test"); + puts "\nRunning new test with absolute correct main and non existing relative test path" + run_espresso_test_with_one_existing_and_one_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{$correct_absolute_mainAPKPath} -PtestAPKPath=#{$incorrect_relative_APKPath}", "test"); + puts "\nRunning new test with relative correct main and non existing absolute test path" + run_espresso_test_with_one_existing_and_one_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{$correct_relative_mainAPKPath} -PtestAPKPath=#{$incorrect_absolute_APKPath}", "test"); + puts "\nRunning new test with relative correct main and non existing relative test path" + run_espresso_test_with_one_existing_and_one_non_existing_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{$correct_relative_mainAPKPath} -PtestAPKPath=#{$incorrect_relative_APKPath}", "test"); +end + +def run_test_with_ipa + puts "Running tests with ipa files" + mainAPKPath = $current_path + "/test/mainApk/ipa" + testAPKPAth = $current_path + "/test/testApk/ipa" + run_espresso_test_with_incorrect_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth}") +end + +def run_test_with_zip + puts "Running tests with ipa files" + mainAPKPath = $current_path + "/test/mainApk/" + testAPKPAth = $current_path + "/test/mainApk/zip" + run_espresso_test_with_one_incorrect_apk_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth}"); +end + def run_tests_with_flavors puts "Running tests with flavors using ./gradlew" run_basic_espresso_test("./gradlew clean executePhoneDebugTestsOnBrowserstack") @@ -126,15 +349,58 @@ def run_cli_test_delete_command(gradle_command) end end +def run_test_with_cucumber_options + puts "\nRunning new test using ./gradlew with absolute APK paths with cucumber options" + mainAPKPath = $current_path + "/test/mainApk/cucumber" + testAPKPAth = $current_path + "/test/testApk/cucumber" + run_espresso_test_with_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth} --config-file=config-browserstack_cucumber.json") + puts "\nRunning new test using ./gradlew with absolute APK paths with invalid cucumber options" + run_espresso_test_with_invalid_cucumber_config("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth} --config-file=config-browserstack_cucumber_invalid_name.json") + puts "\nRunning new test using ./gradlew with relative APK paths with cucumber options" + mainAPKPath = "./test/mainApk/cucumber" + testAPKPAth = "./test/testApk/cucumber" + run_espresso_test_with_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth} --config-file=config-browserstack_cucumber.json") + puts "\nRunning new test using ./gradlew with relative APK paths with invalid cucumber options" + run_espresso_test_with_invalid_cucumber_config("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth} --config-file=config-browserstack_cucumber_invalid_name.json") +end + +def run_test_with_instrumentation_options + puts "\nRunning new test using ./gradlew with absolute APK paths with instrumentation options" + mainAPKPath = $current_path + "/test/mainApk/cucumber" + testAPKPAth = $current_path + "/test/testApk/cucumber" + run_espresso_test_with_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth} --config-file=config-browserstack_instrumentation.json") + puts "\nRunning new test using ./gradlew with absolute APK paths with invalid instrumentation options" + run_espresso_test_with_invalid_instrumentation_config("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth} --config-file=config-browserstack_instrumentation_invalid.json") + puts "\nRunning new test using ./gradlew with relative APK paths with instrumentation options" + mainAPKPath = "./test/mainApk/cucumber" + testAPKPAth = "./test/testApk/cucumber" + run_espresso_test_with_path("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth} --config-file=config-browserstack_instrumentation.json") + puts "\nRunning new test using ./gradlew with relative APK paths with invalid instrumentation options" + run_espresso_test_with_invalid_instrumentation_config("./gradlew executeDebugTestsOnBrowserstack -PmainAPKPath=#{mainAPKPath} -PtestAPKPath=#{testAPKPAth} --config-file=config-browserstack_instrumentation_invalid.json") +end def test validate_env build_plugin setup_repo + setup_required_paths run_tests remove_repo setup_repo + setup_required_paths run_tests_args + run_tests_with_path_args + run_test_with_incorrect_path + run_tests_with_path_variations + run_tests_with_relative_path + run_test_with_incorrect_relative_path + run_tests_with_relative_path_variations + run_test_with_non_existing_path + run_test_with_one_correct_and_one_non_existing_path + run_test_with_ipa + run_test_with_zip + run_test_with_cucumber_options + run_test_with_instrumentation_options run_cli_tests setup_repo_with_app_variants run_tests_with_flavors