SmackIntegrationTestFramework.java

  1. /**
  2.  *
  3.  * Copyright 2015-2023 Florian Schmaus
  4.  *
  5.  * Licensed under the Apache License, Version 2.0 (the "License");
  6.  * you may not use this file except in compliance with the License.
  7.  * You may obtain a copy of the License at
  8.  *
  9.  *     http://www.apache.org/licenses/LICENSE-2.0
  10.  *
  11.  * Unless required by applicable law or agreed to in writing, software
  12.  * distributed under the License is distributed on an "AS IS" BASIS,
  13.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14.  * See the License for the specific language governing permissions and
  15.  * limitations under the License.
  16.  */
  17. package org.igniterealtime.smack.inttest;

  18. import static org.reflections.ReflectionUtils.getAllMethods;
  19. import static org.reflections.ReflectionUtils.withAnnotation;
  20. import static org.reflections.ReflectionUtils.withModifier;
  21. import static org.reflections.ReflectionUtils.withParametersCount;
  22. import static org.reflections.ReflectionUtils.withReturnType;

  23. import java.io.IOException;
  24. import java.lang.annotation.Annotation;
  25. import java.lang.reflect.Constructor;
  26. import java.lang.reflect.InvocationTargetException;
  27. import java.lang.reflect.Method;
  28. import java.lang.reflect.Modifier;
  29. import java.lang.reflect.ParameterizedType;
  30. import java.lang.reflect.Type;
  31. import java.security.KeyManagementException;
  32. import java.security.NoSuchAlgorithmException;
  33. import java.util.ArrayList;
  34. import java.util.Arrays;
  35. import java.util.Collection;
  36. import java.util.Collections;
  37. import java.util.HashMap;
  38. import java.util.HashSet;
  39. import java.util.Iterator;
  40. import java.util.LinkedList;
  41. import java.util.List;
  42. import java.util.Locale;
  43. import java.util.Map;
  44. import java.util.Set;
  45. import java.util.SortedSet;
  46. import java.util.TreeSet;
  47. import java.util.logging.Level;
  48. import java.util.logging.Logger;

  49. import org.jivesoftware.smack.AbstractXMPPConnection;
  50. import org.jivesoftware.smack.ConnectionConfiguration;
  51. import org.jivesoftware.smack.ConnectionConfiguration.SecurityMode;
  52. import org.jivesoftware.smack.Smack;
  53. import org.jivesoftware.smack.SmackConfiguration;
  54. import org.jivesoftware.smack.SmackException;
  55. import org.jivesoftware.smack.SmackException.NoResponseException;
  56. import org.jivesoftware.smack.SmackException.NotConnectedException;
  57. import org.jivesoftware.smack.XMPPException;
  58. import org.jivesoftware.smack.XMPPException.XMPPErrorException;
  59. import org.jivesoftware.smack.tcp.XMPPTCPConnectionConfiguration;
  60. import org.jivesoftware.smack.util.StringUtils;
  61. import org.jivesoftware.smack.util.TLSUtils;
  62. import org.jivesoftware.smack.util.dns.dnsjava.DNSJavaResolver;
  63. import org.jivesoftware.smack.util.dns.javax.JavaxResolver;
  64. import org.jivesoftware.smack.util.dns.minidns.MiniDnsResolver;

  65. import org.jivesoftware.smackx.debugger.EnhancedDebugger;
  66. import org.jivesoftware.smackx.debugger.EnhancedDebuggerWindow;
  67. import org.jivesoftware.smackx.iqregister.AccountManager;

  68. import org.igniterealtime.smack.inttest.Configuration.AccountRegistration;
  69. import org.igniterealtime.smack.inttest.annotations.AfterClass;
  70. import org.igniterealtime.smack.inttest.annotations.BeforeClass;
  71. import org.igniterealtime.smack.inttest.annotations.SmackIntegrationTest;
  72. import org.igniterealtime.smack.inttest.annotations.SpecificationReference;
  73. import org.reflections.Reflections;
  74. import org.reflections.scanners.MethodAnnotationsScanner;
  75. import org.reflections.scanners.MethodParameterScanner;
  76. import org.reflections.scanners.SubTypesScanner;
  77. import org.reflections.scanners.TypeAnnotationsScanner;

  78. public class SmackIntegrationTestFramework {

  79.     static {
  80.         TLSUtils.setDefaultTrustStoreTypeToJksIfRequired();
  81.     }

  82.     private static final Logger LOGGER = Logger.getLogger(SmackIntegrationTestFramework.class.getName());

  83.     public static boolean SINTTEST_UNIT_TEST = false;

  84.     private static ConcreteTest TEST_UNDER_EXECUTION;

  85.     protected final Configuration config;

  86.     protected TestRunResult testRunResult;

  87.     private SmackIntegrationTestEnvironment environment;
  88.     protected XmppConnectionManager connectionManager;

  89.     public enum TestType {
  90.         Normal,
  91.         LowLevel,
  92.         SpecificLowLevel,
  93.     }

  94.     public static void main(String[] args) throws IOException, KeyManagementException,
  95.             NoSuchAlgorithmException, SmackException, XMPPException, InterruptedException, InstantiationException,
  96.             IllegalAccessException, IllegalArgumentException, InvocationTargetException {
  97.         Configuration config = Configuration.newConfiguration(args);

  98.         SmackIntegrationTestFramework sinttest = new SmackIntegrationTestFramework(config);
  99.         TestRunResult testRunResult = sinttest.run();

  100.         for (final TestRunResultProcessor testRunResultProcessor : config.testRunResultProcessors) {
  101.             try {
  102.                 testRunResultProcessor.process(testRunResult);
  103.             } catch (Throwable t) {
  104.                 LOGGER.log(Level.WARNING, "Invocation of TestRunResultProcessor " + testRunResultProcessor + " failed.", t);
  105.             }
  106.         }

  107.         if (config.debuggerFactory instanceof EnhancedDebugger) {
  108.             EnhancedDebuggerWindow.getInstance().waitUntilClosed();
  109.         }

  110.         final int exitStatus = testRunResult.failedIntegrationTests.isEmpty() ? 0 : 2;
  111.         System.exit(exitStatus);
  112.     }

  113.     public static class JulTestRunResultProcessor implements TestRunResultProcessor {

  114.         @Override
  115.         public void process(final TestRunResult testRunResult) {
  116.             for (Map.Entry<Class<? extends AbstractSmackIntTest>, Throwable> entry : testRunResult.impossibleTestClasses.entrySet()) {
  117.                 LOGGER.info("Could not run " + entry.getKey().getName() + " because: "
  118.                     + entry.getValue().getLocalizedMessage());
  119.             }
  120.             for (TestNotPossible testNotPossible : testRunResult.impossibleIntegrationTests) {
  121.                 LOGGER.info("Could not run " + testNotPossible.concreteTest + " because: "
  122.                     + testNotPossible.testNotPossibleException.getMessage());
  123.             }
  124.             for (SuccessfulTest successfulTest : testRunResult.successfulIntegrationTests) {
  125.                 LOGGER.info(successfulTest.concreteTest + " ✔");
  126.             }
  127.             final int successfulTests = testRunResult.successfulIntegrationTests.size();
  128.             final int failedTests = testRunResult.failedIntegrationTests.size();
  129.             final int availableTests = testRunResult.getNumberOfAvailableTests();
  130.             LOGGER.info("SmackIntegrationTestFramework[" + testRunResult.testRunId + ']' + " finished: "
  131.                 + successfulTests + '/' + availableTests + " [" + failedTests + " failed]");

  132.             if (failedTests > 0) {
  133.                 LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀");
  134.                 final SortedSet<String> bySpecification = new TreeSet<>();
  135.                 for (FailedTest failedTest : testRunResult.failedIntegrationTests) {
  136.                     final Throwable cause = failedTest.failureReason;
  137.                     LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause);
  138.                 if (failedTest.concreteTest.method.getDeclaringClass().isAnnotationPresent(SpecificationReference.class)) {
  139.                         final String specificationReference = getSpecificationReference(failedTest.concreteTest.method);
  140.                         if (specificationReference != null) {
  141.                             bySpecification.add("- " + specificationReference + " (as tested by '" + failedTest.concreteTest + "')");
  142.                         }
  143.                     }
  144.                 }
  145.                 if (!bySpecification.isEmpty()) {
  146.                     LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification));
  147.                 }
  148.             } else {
  149.                 LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/");
  150.             }
  151.         }
  152.     }

  153.     private static String getSpecificationReference(Method method) {
  154.         final SpecificationReference spec = method.getDeclaringClass().getAnnotation(SpecificationReference.class);
  155.         if (spec == null || spec.document().isBlank()) {
  156.             return null;
  157.         }
  158.         String line = spec.document().trim();
  159.         if (!spec.version().isBlank()) {
  160.             line += " (version " + spec.version() + ")";
  161.         }

  162.         final SmackIntegrationTest test = method.getAnnotation(SmackIntegrationTest.class);
  163.         if (!test.section().isBlank()) {
  164.             line += " section " + test.section().trim();
  165.         }
  166.         if (!test.quote().isBlank()) {
  167.             line += ":\t\"" + test.quote().trim() + "\"";
  168.         }
  169.         assert !line.isBlank();
  170.         return line;
  171.     }

  172.     public SmackIntegrationTestFramework(Configuration configuration) {
  173.         this.config = configuration;
  174.     }

  175.     public synchronized TestRunResult run()
  176.             throws KeyManagementException, NoSuchAlgorithmException, SmackException, IOException, XMPPException,
  177.             InterruptedException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
  178.         // The DNS resolver is not really a per sinttest run setting. It is not even a per connection setting. Instead
  179.         // it is a global setting, but we treat it like a per sinttest run setting.
  180.         switch (config.dnsResolver) {
  181.         case minidns:
  182.             MiniDnsResolver.setup();
  183.             break;
  184.         case javax:
  185.             JavaxResolver.setup();
  186.             break;
  187.         case dnsjava:
  188.             DNSJavaResolver.setup();
  189.             break;
  190.         }
  191.         testRunResult = new TestRunResult();

  192.         // Create a connection manager *after* we created the testRunId (in testRunResult).
  193.         this.connectionManager = new XmppConnectionManager(this);

  194.         LOGGER.info("SmackIntegrationTestFramework [" + testRunResult.testRunId + ']' + ": Starting\nSmack version: " + Smack.getVersion());
  195.         if (config.debuggerFactory != null) {
  196.             // JUL Debugger will not print any information until configured to print log messages of
  197.             // level FINE
  198.             // TODO configure JUL for log?
  199.             SmackConfiguration.addDisabledSmackClass("org.jivesoftware.smack.debugger.JulDebugger");
  200.             SmackConfiguration.DEBUG = true;
  201.         }
  202.         if (config.replyTimeout > 0) {
  203.             SmackConfiguration.setDefaultReplyTimeout(config.replyTimeout);
  204.         }
  205.         if (config.securityMode != SecurityMode.required && config.accountRegistration == AccountRegistration.inBandRegistration) {
  206.             AccountManager.sensitiveOperationOverInsecureConnectionDefault(true);
  207.         }
  208.         // TODO print effective configuration

  209.         String[] testPackages;
  210.         if (config.testPackages == null || config.testPackages.isEmpty()) {
  211.             testPackages = new String[] { "org.jivesoftware.smackx", "org.jivesoftware.smack", "org.igniterealtime.smackx", "org.igniterealtime.smack" };
  212.         }
  213.         else {
  214.             testPackages = config.testPackages.toArray(new String[config.testPackages.size()]);
  215.         }
  216.         Reflections reflections = new Reflections(testPackages, new SubTypesScanner(),
  217.                         new TypeAnnotationsScanner(), new MethodAnnotationsScanner(), new MethodParameterScanner());
  218.         Set<Class<? extends AbstractSmackIntegrationTest>> inttestClasses = reflections.getSubTypesOf(AbstractSmackIntegrationTest.class);
  219.         Set<Class<? extends AbstractSmackLowLevelIntegrationTest>> lowLevelInttestClasses = reflections.getSubTypesOf(AbstractSmackLowLevelIntegrationTest.class);

  220.         final int builtInTestCount = inttestClasses.size() + lowLevelInttestClasses.size();
  221.         Set<Class<? extends AbstractSmackIntTest>> classes = new HashSet<>(builtInTestCount);
  222.         classes.addAll(inttestClasses);
  223.         classes.addAll(lowLevelInttestClasses);

  224.         {
  225.             // Remove all abstract classes.
  226.             // TODO: This may be a good candidate for Java stream filtering once Smack is Android API 24 or higher.
  227.             Iterator<Class<? extends AbstractSmackIntTest>> it = classes.iterator();
  228.             while (it.hasNext()) {
  229.                 Class<? extends AbstractSmackIntTest> clazz = it.next();
  230.                 if (Modifier.isAbstract(clazz.getModifiers())) {
  231.                     it.remove();
  232.                 }
  233.             }
  234.         }

  235.         if (classes.isEmpty()) {
  236.             throw new IllegalStateException("No test classes in " + Arrays.toString(testPackages) + " found");
  237.         }

  238.         LOGGER.info("SmackIntegrationTestFramework [" + testRunResult.testRunId
  239.                         + "]: Finished scanning for tests, preparing environment\n"
  240.                         + "\tJava SE Platform version: " + Runtime.version());
  241.         environment = prepareEnvironment();

  242.         try {
  243.             runTests(classes);
  244.         }
  245.         catch (Throwable t) {
  246.             // Log the thrown Throwable to prevent it being shadowed in case the finally block below also throws.
  247.             LOGGER.log(Level.SEVERE, "Unexpected abort because runTests() threw throwable", t);
  248.             throw t;
  249.         }
  250.         finally {
  251.             // Ensure that the accounts are deleted and disconnected before we continue
  252.             connectionManager.disconnectAndCleanup();
  253.         }

  254.         return testRunResult;
  255.     }

  256.     public static ConcreteTest getTestUnderExecution() {
  257.         return TEST_UNDER_EXECUTION;
  258.     }

  259.     @SuppressWarnings({"Finally"})
  260.     private void runTests(Set<Class<? extends AbstractSmackIntTest>> classes)
  261.             throws InterruptedException, InstantiationException, IllegalAccessException,
  262.             IllegalArgumentException, SmackException, IOException, XMPPException {
  263.         List<PreparedTest> tests = new ArrayList<>(classes.size());
  264.         int numberOfAvailableTests = 0;

  265.         for (Class<? extends AbstractSmackIntTest> testClass : classes) {
  266.             final String testClassName = testClass.getName();

  267.             // TODO: Move the whole "skipping section" below one layer up?

  268.             // Skip pseudo integration tests from src/test
  269.             // Although Smack's gradle build files do not state that the 'main' sources classpath also contains the
  270.             // 'test' classes. Some IDEs like Eclipse include them. As result, a real integration test run encounters
  271.             // pseudo integration tests like the DummySmackIntegrationTest which always throws from src/test.
  272.             // It is unclear why this apparently does not happen in the 4.3 branch, one likely cause is
  273.             // compile project(path: ":smack-omemo", configuration: "testRuntime")
  274.             // in
  275.             // smack-integration-test/build.gradle:17
  276.             // added after 4.3 was branched out with
  277.             // 1f731f6318785a84b9741280d586a61dc37ecb2e
  278.             // Now "gradle integrationTest" appear to be never affected by this, i.e., they are executed with the
  279.             // correct classpath. Plain Eclipse, i.e. Smack imported into Eclipse after "gradle eclipse", appear
  280.             // to include *all* classes. Which means those runs sooner or later try to execute
  281.             // DummySmackIntegrationTest. Eclipse with buildship, the gradle plugin for Eclipse, always excludes
  282.             // *all* src/test classes, which means they do not encounter DummySmackIntegrationTest, but this means
  283.             // that the "compile project(path: ":smack-omemo", configuration: "testRuntime")" is not respected,
  284.             // which leads to
  285.             // Exception in thread "main" java.lang.NoClassDefFoundError: org/jivesoftware/smack/test/util/FileTestUtil
  286.             //   at org.jivesoftware.smackx.ox.OXSecretKeyBackupIntegrationTest.<clinit>(OXSecretKeyBackupIntegrationTest.java:66)
  287.             // See
  288.             // - https://github.com/eclipse/buildship/issues/354 (Remove test dependencies from runtime classpath)
  289.             // - https://bugs.eclipse.org/bugs/show_bug.cgi?id=482315 (Runtime classpath includes test dependencies)
  290.             // - https://discuss.gradle.org/t/main-vs-test-compile-vs-runtime-classpaths-in-eclipse-once-and-for-all-how/17403
  291.             // - https://bugs.eclipse.org/bugs/show_bug.cgi?id=376616 (Scope of dependencies has no effect on Eclipse compilation)
  292.             if (!SINTTEST_UNIT_TEST && testClassName.startsWith("org.igniterealtime.smack.inttest.unittest")) {
  293.                 LOGGER.warning("Skipping integration test '" + testClassName + "' from src/test classpath (should not be in classpath)");
  294.                 continue;
  295.             }

  296.             if (!config.isClassEnabled(testClass)) {
  297.                 DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test class " + testClassName + " because it is not enabled");
  298.                 testRunResult.disabledTestClasses.add(disabledTestClass);
  299.                 continue;
  300.             }

  301.             if (config.isClassDisabled(testClass)) {
  302.                 DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test class " + testClassName + " because it is disabled");
  303.                 testRunResult.disabledTestClasses.add(disabledTestClass);
  304.                 continue;
  305.             }

  306.             final String specification;
  307.             if (testClass.isAnnotationPresent(SpecificationReference.class)) {
  308.                 final SpecificationReference specificationReferenceAnnotation = testClass.getAnnotation(SpecificationReference.class);
  309.                 specification = Configuration.normalizeSpecification(specificationReferenceAnnotation.document());
  310.             } else {
  311.                 specification = null;
  312.             }

  313.             if (!config.isSpecificationEnabled(specification)) {
  314.                 DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test method " + testClass + " because it tests a specification ('" + specification + "') that is not enabled");
  315.                 testRunResult.disabledTestClasses.add(disabledTestClass);
  316.                 continue;
  317.             }

  318.             if (config.isSpecificationDisabled(specification)) {
  319.                 DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test method " + testClass + " because it tests a specification ('" + specification + "') that is disabled");
  320.                 testRunResult.disabledTestClasses.add(disabledTestClass);
  321.                 continue;
  322.             }

  323.             final Constructor<? extends AbstractSmackIntTest> cons;
  324.             try {
  325.                 cons = testClass.getConstructor(SmackIntegrationTestEnvironment.class);
  326.             }
  327.             catch (NoSuchMethodException | SecurityException e) {
  328.                 throw new IllegalArgumentException(
  329.                                 "Smack Integration Test class does not declare the correct constructor. Is a public Constructor(SmackIntegrationTestEnvironment) missing?",
  330.                                 e);
  331.             }

  332.             final List<Method> smackIntegrationTestMethods;
  333.             {
  334.                 Method[] testClassMethods = testClass.getMethods();
  335.                 smackIntegrationTestMethods = new ArrayList<>(testClassMethods.length);
  336.                 for (Method method : testClassMethods) {
  337.                     if (!method.isAnnotationPresent(SmackIntegrationTest.class)) {
  338.                         continue;
  339.                     }
  340.                     smackIntegrationTestMethods.add(method);
  341.                 }
  342.             }

  343.             if (smackIntegrationTestMethods.isEmpty()) {
  344.                 LOGGER.warning("No Smack integration test methods found in " + testClass);
  345.                 continue;
  346.             }

  347.             final AbstractSmackIntTest test;
  348.             try {
  349.                 test = cons.newInstance(environment);
  350.             }
  351.             catch (InvocationTargetException e) {
  352.                 Throwable cause = e.getCause();

  353.                 throwFatalException(cause);

  354.                 testRunResult.impossibleTestClasses.put(testClass, cause);
  355.                 continue;
  356.             }

  357.             XmppConnectionDescriptor<?, ?, ?> specificLowLevelConnectionDescriptor = null;
  358.             final TestType testType;
  359.             if (test instanceof AbstractSmackSpecificLowLevelIntegrationTest) {
  360.                 AbstractSmackSpecificLowLevelIntegrationTest<?> specificLowLevelTest = (AbstractSmackSpecificLowLevelIntegrationTest<?>) test;
  361.                 specificLowLevelConnectionDescriptor = specificLowLevelTest.getConnectionDescriptor();
  362.                 testType = TestType.SpecificLowLevel;
  363.             } else if (test instanceof AbstractSmackLowLevelIntegrationTest) {
  364.                 testType = TestType.LowLevel;
  365.             } else if (test instanceof AbstractSmackIntegrationTest) {
  366.                 testType = TestType.Normal;
  367.             } else {
  368.                 throw new AssertionError();
  369.             }

  370.             // Verify the method signatures, throw in case a signature is incorrect.
  371.             for (Method method : smackIntegrationTestMethods) {
  372.                 Class<?> retClass = method.getReturnType();
  373.                 if (!retClass.equals(Void.TYPE)) {
  374.                     throw new IllegalStateException(
  375.                             "SmackIntegrationTest annotation on" + method + " that does not return void");
  376.                 }
  377.                 switch (testType) {
  378.                 case Normal:
  379.                     final Class<?>[] parameterTypes = method.getParameterTypes();
  380.                     if (parameterTypes.length > 0) {
  381.                         throw new IllegalStateException(
  382.                                 "SmackIntegrationTest annotation on " + method + " that takes arguments ");
  383.                     }
  384.                     break;
  385.                 case LowLevel:
  386.                     verifyLowLevelTestMethod(method, AbstractXMPPConnection.class);
  387.                     break;
  388.                 case SpecificLowLevel:
  389.                     Class<? extends AbstractXMPPConnection> specificLowLevelConnectionClass = specificLowLevelConnectionDescriptor.getConnectionClass();
  390.                     verifyLowLevelTestMethod(method, specificLowLevelConnectionClass);
  391.                     break;
  392.                 }
  393.             }

  394.             Iterator<Method> it = smackIntegrationTestMethods.iterator();
  395.             while (it.hasNext()) {
  396.                 final Method method = it.next();
  397.                 final String methodName = method.getName();
  398.                 if (!config.isMethodEnabled(method)) {
  399.                     DisabledTest disabledTest = new DisabledTest(method, "Skipping test method " + methodName + " because it is not enabled");
  400.                     testRunResult.disabledTests.add(disabledTest);
  401.                     it.remove();
  402.                     continue;
  403.                 }
  404.                 if (config.isMethodDisabled(method)) {
  405.                     DisabledTest disabledTest = new DisabledTest(method, "Skipping test method " + methodName + " because it is disabled");
  406.                     testRunResult.disabledTests.add(disabledTest);
  407.                     it.remove();
  408.                     continue;
  409.                 }
  410.             }

  411.             if (smackIntegrationTestMethods.isEmpty()) {
  412.                 LOGGER.info("All tests in " + testClassName + " are disabled");
  413.                 continue;
  414.             }

  415.             List<ConcreteTest> concreteTests = new ArrayList<>(smackIntegrationTestMethods.size());

  416.             for (Method testMethod : smackIntegrationTestMethods) {
  417.                 switch (testType) {
  418.                 case Normal: {
  419.                     ConcreteTest.Executor concreteTestExecutor = () -> testMethod.invoke(test);
  420.                     ConcreteTest concreteTest = new ConcreteTest(testType, testMethod, concreteTestExecutor);
  421.                     concreteTests.add(concreteTest);
  422.                 }
  423.                     break;
  424.                 case LowLevel:
  425.                 case SpecificLowLevel:
  426.                     LowLevelTestMethod lowLevelTestMethod = new LowLevelTestMethod(testMethod);
  427.                     switch (testType) {
  428.                     case LowLevel:
  429.                         List<ConcreteTest> concreteLowLevelTests = invokeLowLevel(lowLevelTestMethod, (AbstractSmackLowLevelIntegrationTest) test);
  430.                         concreteTests.addAll(concreteLowLevelTests);
  431.                         break;
  432.                     case SpecificLowLevel: {
  433.                         ConcreteTest.Executor concreteTestExecutor = () -> invokeSpecificLowLevel(
  434.                                 lowLevelTestMethod, (AbstractSmackSpecificLowLevelIntegrationTest<?>) test);
  435.                         ConcreteTest concreteTest = new ConcreteTest(testType, testMethod, concreteTestExecutor);
  436.                         concreteTests.add(concreteTest);
  437.                         break;
  438.                     }
  439.                     default:
  440.                         throw new AssertionError();
  441.                     }
  442.                     break;
  443.                 }
  444.             }

  445.             // Instantiate the prepared test early as this will check the before and after class annotations.
  446.             PreparedTest preparedTest = new PreparedTest(test, concreteTests);
  447.             tests.add(preparedTest);

  448.             numberOfAvailableTests += concreteTests.size();
  449.         }

  450.         // Print status information.
  451.         StringBuilder sb = new StringBuilder(1024);
  452.         sb.append("Smack Integration Test Framework\n");
  453.         sb.append("################################\n");
  454.         if (config.verbose) {
  455.             sb.append('\n');
  456.             if (!testRunResult.disabledTestClasses.isEmpty()) {
  457.                 sb.append("The following test classes are disabled:\n");
  458.                 for (DisabledTestClass disabledTestClass : testRunResult.disabledTestClasses) {
  459.                     disabledTestClass.appendTo(sb).append('\n');
  460.                 }
  461.             }
  462.             if (!testRunResult.disabledTests.isEmpty()) {
  463.                 sb.append("The following tests are disabled:\n");
  464.                 for (DisabledTest disabledTest : testRunResult.disabledTests) {
  465.                     disabledTest.appendTo(sb).append('\n');
  466.                 }
  467.             }
  468.             sb.append('\n');
  469.         }

  470.         if (numberOfAvailableTests == 0) {
  471.             throw new IllegalArgumentException("No integration tests selected.");
  472.         }

  473.         sb.append("Available tests: ").append(numberOfAvailableTests);
  474.         if (!testRunResult.disabledTestClasses.isEmpty() || !testRunResult.disabledTests.isEmpty()) {
  475.             sb.append(" (Disabled ").append(testRunResult.disabledTestClasses.size()).append(" classes")
  476.               .append(" and ").append(testRunResult.disabledTests.size()).append(" tests)");
  477.         }
  478.         sb.append('\n');
  479.         LOGGER.info(sb.toString());

  480.         for (PreparedTest test : tests) {
  481.             test.run();
  482.         }

  483.         // Assert that all tests in the 'tests' list produced a result.
  484.         assert numberOfAvailableTests == testRunResult.getNumberOfAvailableTests();
  485.     }

  486.     private void runConcreteTest(ConcreteTest concreteTest)
  487.             throws InterruptedException, XMPPException, IOException, SmackException {
  488.         LOGGER.info(concreteTest + " Start");
  489.         long testStart = System.currentTimeMillis();
  490.         try {
  491.             concreteTest.executor.execute();
  492.             long testEnd = System.currentTimeMillis();
  493.             LOGGER.info(concreteTest + " Success");
  494.             testRunResult.successfulIntegrationTests.add(new SuccessfulTest(concreteTest, testStart, testEnd, null));
  495.         }
  496.         catch (InvocationTargetException e) {
  497.             long testEnd = System.currentTimeMillis();
  498.             Throwable cause = e.getCause();
  499.             if (cause instanceof TestNotPossibleException) {
  500.                 LOGGER.info(concreteTest + " is not possible");
  501.                 testRunResult.impossibleIntegrationTests.add(new TestNotPossible(concreteTest, testStart, testEnd,
  502.                                 null, (TestNotPossibleException) cause));
  503.                 return;
  504.             }
  505.             Throwable nonFatalFailureReason;
  506.             // junit asserts throw an AssertionError if they fail, those should not be
  507.             // thrown up, as it would be done by throwFatalException()
  508.             if (cause instanceof AssertionError) {
  509.                 nonFatalFailureReason = cause;
  510.             } else {
  511.                 nonFatalFailureReason = throwFatalException(cause);
  512.             }
  513.             // An integration test failed
  514.             testRunResult.failedIntegrationTests.add(new FailedTest(concreteTest, testStart, testEnd, null,
  515.                             nonFatalFailureReason));
  516.             LOGGER.log(Level.SEVERE, concreteTest + " Failed", e);
  517.         }
  518.         catch (IllegalArgumentException | IllegalAccessException e) {
  519.             throw new AssertionError(e);
  520.         }
  521.     }

  522.     private static void verifyLowLevelTestMethod(Method method,
  523.                     Class<? extends AbstractXMPPConnection> connectionClass) {
  524.         if (determineTestMethodParameterType(method, connectionClass) != null) {
  525.             return;
  526.         }

  527.         throw new IllegalArgumentException(method + " is not a valid low level test method");
  528.     }

  529.     private List<ConcreteTest> invokeLowLevel(LowLevelTestMethod lowLevelTestMethod, AbstractSmackLowLevelIntegrationTest test) {
  530.         Collection<? extends XmppConnectionDescriptor<?, ?, ?>> connectionDescriptors;
  531.         if (lowLevelTestMethod.smackIntegrationTestAnnotation.onlyDefaultConnectionType()) {
  532.             XmppConnectionDescriptor<?, ?, ?> defaultConnectionDescriptor = connectionManager.getDefaultConnectionDescriptor();
  533.             connectionDescriptors = Collections.singleton(defaultConnectionDescriptor);
  534.         } else {
  535.             connectionDescriptors = connectionManager.getConnectionDescriptors();
  536.         }

  537.         List<ConcreteTest> resultingConcreteTests = new ArrayList<>(connectionDescriptors.size());

  538.         for (XmppConnectionDescriptor<?, ?, ?> connectionDescriptor : connectionDescriptors) {
  539.             String connectionNick = connectionDescriptor.getNickname();

  540.             if (config.enabledConnections != null && !config.enabledConnections.contains(connectionNick)) {
  541.                 DisabledTest disabledTest = new DisabledTest(lowLevelTestMethod.testMethod, "Not creating test for " + lowLevelTestMethod + " with connection '" + connectionNick
  542.                                 + "', as this connection type is not enabled");
  543.                 testRunResult.disabledTests.add(disabledTest);
  544.                 continue;
  545.             }

  546.             if (config.disabledConnections != null && config.disabledConnections.contains(connectionNick)) {
  547.                 DisabledTest disabledTest = new DisabledTest(lowLevelTestMethod.testMethod, "Not creating test for " + lowLevelTestMethod + " with connection '" + connectionNick
  548.                                 + ", as this connection type is disabled");
  549.                 testRunResult.disabledTests.add(disabledTest);
  550.                 continue;
  551.             }

  552.             ConcreteTest.Executor executor = () -> lowLevelTestMethod.invoke(test, connectionDescriptor);
  553.             ConcreteTest concreteTest = new ConcreteTest(TestType.LowLevel, lowLevelTestMethod.testMethod, executor, connectionDescriptor.getNickname());
  554.             resultingConcreteTests.add(concreteTest);
  555.         }

  556.         return resultingConcreteTests;
  557.     }

  558.     private static <C extends AbstractXMPPConnection> void invokeSpecificLowLevel(LowLevelTestMethod testMethod,
  559.             AbstractSmackSpecificLowLevelIntegrationTest<C> test)
  560.             throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, InterruptedException,
  561.             SmackException, IOException, XMPPException {
  562.         if (testMethod.smackIntegrationTestAnnotation.onlyDefaultConnectionType()) {
  563.             throw new IllegalArgumentException("SpecificLowLevelTests must not have set onlyDefaultConnectionType");
  564.         }

  565.         XmppConnectionDescriptor<C, ? extends ConnectionConfiguration, ? extends ConnectionConfiguration.Builder<?, ?>> connectionDescriptor = test.getConnectionDescriptor();
  566.         testMethod.invoke(test, connectionDescriptor);
  567.     }

  568.     protected SmackIntegrationTestEnvironment prepareEnvironment() throws SmackException,
  569.                     IOException, XMPPException, InterruptedException, KeyManagementException,
  570.                     NoSuchAlgorithmException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
  571.         return connectionManager.prepareEnvironment();
  572.     }

  573.     enum AccountNum {
  574.         One,
  575.         Two,
  576.         Three,
  577.     }

  578.     static XMPPTCPConnectionConfiguration.Builder getConnectionConfigurationBuilder(Configuration config) {
  579.         XMPPTCPConnectionConfiguration.Builder builder = XMPPTCPConnectionConfiguration.builder();

  580.         config.configurationApplier.applyConfigurationTo(builder);

  581.         return builder;
  582.     }

  583.     private static Exception throwFatalException(Throwable e) throws Error, NoResponseException,
  584.                     InterruptedException {
  585.         if (e instanceof InterruptedException) {
  586.             throw (InterruptedException) e;
  587.         }

  588.         // We handle NullPointerException as a non-fatal exception, as they are mostly caused by an invalid reply where
  589.         // an extension element is missing. Consider for example
  590.         // assertEquals(StanzaError.Condition.foo, response.getError().getCondition())
  591.         // Otherwise NPEs could be caused by an internal bug in Smack, e.g. missing null handling.
  592.         if (e instanceof NullPointerException) {
  593.             return (NullPointerException) e;
  594.         }
  595.         if (e instanceof RuntimeException) {
  596.             throw (RuntimeException) e;
  597.         }
  598.         if (e instanceof Error) {
  599.             throw (Error) e;
  600.         }
  601.         return (Exception) e;
  602.     }

  603.     @FunctionalInterface
  604.     public interface TestRunResultProcessor {
  605.         void process(SmackIntegrationTestFramework.TestRunResult testRunResult);
  606.     }

  607.     public static final class TestRunResult {

  608.         /**
  609.          * A short String of lowercase characters and numbers used to identify an integration test
  610.          * run. We use lowercase characters because this string will eventually be part of the
  611.          * localpart of the used JIDs (and the localpart is case-insensitive).
  612.          */
  613.         public final String testRunId = StringUtils.insecureRandomString(5).toLowerCase(Locale.US);

  614.         private final List<SuccessfulTest> successfulIntegrationTests = Collections.synchronizedList(new LinkedList<SuccessfulTest>());
  615.         private final List<FailedTest> failedIntegrationTests = Collections.synchronizedList(new LinkedList<FailedTest>());
  616.         private final List<TestNotPossible> impossibleIntegrationTests = Collections.synchronizedList(new LinkedList<TestNotPossible>());

  617.         // TODO: Ideally three would only be a list of disabledTests, but since we do not process a disabled test class
  618.         // any further, we can not determine the concrete disabled tests.
  619.         private final List<DisabledTestClass> disabledTestClasses = Collections.synchronizedList(new ArrayList<>());
  620.         private final List<DisabledTest> disabledTests = Collections.synchronizedList(new ArrayList<>());

  621.         private final Map<Class<? extends AbstractSmackIntTest>, Throwable> impossibleTestClasses = new HashMap<>();

  622.         TestRunResult() {
  623.         }

  624.         public String getTestRunId() {
  625.             return testRunId;
  626.         }

  627.         public int getNumberOfAvailableTests() {
  628.             return successfulIntegrationTests.size() + failedIntegrationTests.size() + impossibleIntegrationTests.size();
  629.         }

  630.         public List<SuccessfulTest> getSuccessfulTests() {
  631.             return Collections.unmodifiableList(successfulIntegrationTests);
  632.         }

  633.         public List<FailedTest> getFailedTests() {
  634.             return Collections.unmodifiableList(failedIntegrationTests);
  635.         }

  636.         public List<TestNotPossible> getNotPossibleTests() {
  637.             return Collections.unmodifiableList(impossibleIntegrationTests);
  638.         }

  639.         public Map<Class<? extends AbstractSmackIntTest>, Throwable> getImpossibleTestClasses() {
  640.             return Collections.unmodifiableMap(impossibleTestClasses);
  641.         }
  642.     }

  643.     final class PreparedTest {
  644.         private final AbstractSmackIntTest test;
  645.         private final List<ConcreteTest> concreteTests;

  646.         private final Method beforeClassMethod;
  647.         private final Method afterClassMethod;

  648.         private PreparedTest(AbstractSmackIntTest test, List<ConcreteTest> concreteTests) {
  649.             this.test = test;
  650.             this.concreteTests = concreteTests;
  651.             Class<? extends AbstractSmackIntTest> testClass = test.getClass();

  652.             beforeClassMethod = getSinttestSpecialMethod(testClass, BeforeClass.class);
  653.             afterClassMethod = getSinttestSpecialMethod(testClass, AfterClass.class);
  654.         }

  655.         public void run() throws InterruptedException, XMPPException, IOException, SmackException {
  656.             try {
  657.                 // Run the @BeforeClass methods (if any)
  658.                 executeSinttestSpecialMethod(beforeClassMethod);

  659.                 for (ConcreteTest concreteTest : concreteTests) {
  660.                     TEST_UNDER_EXECUTION = concreteTest;
  661.                     try {
  662.                         runConcreteTest(concreteTest);
  663.                     } finally {
  664.                         TEST_UNDER_EXECUTION = null;
  665.                     }
  666.                 }
  667.             }
  668.             finally {
  669.                 executeSinttestSpecialMethod(afterClassMethod);
  670.             }
  671.         }

  672.         private void executeSinttestSpecialMethod(Method method) {
  673.             if (method == null) {
  674.                 return;
  675.             }

  676.             try {
  677.                 method.invoke(test);
  678.             }
  679.             catch (InvocationTargetException | IllegalAccessException e) {
  680.                 LOGGER.log(Level.SEVERE, "Exception executing " + method, e);
  681.             }
  682.             catch (IllegalArgumentException e) {
  683.                 throw new AssertionError(e);
  684.             }
  685.         }
  686.     }

  687.     @SuppressWarnings("unchecked")
  688.     private static Method getSinttestSpecialMethod(Class<? extends AbstractSmackIntTest> testClass, Class<? extends Annotation> annotation) {
  689.         Set<Method> specialClassMethods = getAllMethods(testClass,
  690.                         withAnnotation(annotation), withReturnType(Void.TYPE),
  691.                         withParametersCount(0), withModifier(Modifier.PUBLIC
  692.                                         ));

  693.         // See if there are any methods that have a special but a wrong signature
  694.         Set<Method> allSpecialClassMethods = getAllMethods(testClass, withAnnotation(annotation));
  695.         allSpecialClassMethods.removeAll(specialClassMethods);
  696.         if (!allSpecialClassMethods.isEmpty()) {
  697.             throw new IllegalArgumentException(annotation + " methods with wrong signature found");
  698.         }

  699.         if (specialClassMethods.size() == 1) {
  700.             return specialClassMethods.iterator().next();
  701.         }
  702.         else if (specialClassMethods.size() > 1) {
  703.             throw new IllegalArgumentException("Only one @BeforeClass method allowed");
  704.         }

  705.         return null;
  706.     }

  707.     public static final class ConcreteTest {
  708.         private final TestType testType;
  709.         private final Method method;
  710.         private final Executor executor;
  711.         private final List<String> subdescriptons;

  712.         private ConcreteTest(TestType testType, Method method, Executor executor, String... subdescriptions) {
  713.             this.testType = testType;
  714.             this.method = method;
  715.             this.executor = executor;
  716.             this.subdescriptons = List.of(subdescriptions);
  717.         }

  718.         private transient String stringCache;

  719.         public TestType getTestType() {
  720.             return testType;
  721.         }

  722.         public Method getMethod() {
  723.             return method;
  724.         }

  725.         public List<String> getSubdescriptons() {
  726.             return subdescriptons;
  727.         }

  728.         @Override
  729.         public String toString() {
  730.             if (stringCache != null) {
  731.                 return stringCache;
  732.             }

  733.             StringBuilder sb = new StringBuilder();
  734.             sb.append(method.getDeclaringClass().getSimpleName())
  735.                 .append('.')
  736.                 .append(method.getName())
  737.                 .append(" (")
  738.                 .append(testType.name());
  739.             if (!subdescriptons.isEmpty()) {
  740.                 sb.append(", ");
  741.                 StringUtils.appendTo(subdescriptons, sb);
  742.             }
  743.             sb.append(')');

  744.             stringCache = sb.toString();
  745.             return stringCache;
  746.         }

  747.         private interface Executor {

  748.             /**
  749.              * Execute the test.
  750.              *
  751.              * @throws IllegalAccessException if there was an illegal access.
  752.              * @throws InterruptedException if the calling thread was interrupted.
  753.              * @throws InvocationTargetException if the reflective invoked test throws an exception.
  754.              * @throws XMPPException in case an XMPPException happens when <em>preparing</em> the test.
  755.              * @throws IOException in case an IOException happens when <em>preparing</em> the test.
  756.              * @throws SmackException in case an SmackException happens when <em>preparing</em> the test.
  757.              */
  758.             void execute() throws IllegalAccessException, InterruptedException, InvocationTargetException,
  759.                     XMPPException, IOException, SmackException;
  760.         }
  761.     }

  762.     public static final class DisabledTestClass {
  763.         private final Class<? extends AbstractSmackIntTest> testClass;
  764.         private final String reason;

  765.         private DisabledTestClass(Class<? extends AbstractSmackIntTest> testClass, String reason) {
  766.             this.testClass = testClass;
  767.             this.reason = reason;
  768.         }

  769.         public Class<? extends AbstractSmackIntTest> getTestClass() {
  770.             return testClass;
  771.         }

  772.         public String getReason() {
  773.             return reason;
  774.         }

  775.         public StringBuilder appendTo(StringBuilder sb) {
  776.             return sb.append("Disabled ").append(testClass).append(" because ").append(reason);
  777.         }
  778.     }

  779.     public static final class DisabledTest {
  780.         private final Method method;
  781.         private final String reason;

  782.         private DisabledTest(Method method, String reason) {
  783.             this.method = method;
  784.             this.reason = reason;
  785.         }

  786.         public Method getMethod() {
  787.             return method;
  788.         }

  789.         public String getReason() {
  790.             return reason;
  791.         }

  792.         public StringBuilder appendTo(StringBuilder sb) {
  793.             return sb.append("Disabled ").append(method).append(" because ").append(reason);
  794.         }
  795.     }

  796.     private final class LowLevelTestMethod {

  797.         private final Method testMethod;
  798.         private final SmackIntegrationTest smackIntegrationTestAnnotation;
  799.         private final TestMethodParameterType parameterType;

  800.         private LowLevelTestMethod(Method testMethod) {
  801.             this.testMethod = testMethod;

  802.             smackIntegrationTestAnnotation = testMethod.getAnnotation(SmackIntegrationTest.class);
  803.             assert smackIntegrationTestAnnotation != null;
  804.             parameterType = determineTestMethodParameterType(testMethod);
  805.         }

  806.         private void invoke(AbstractSmackLowLevelIntegrationTest test,
  807.                         XmppConnectionDescriptor<?, ?, ?> connectionDescriptor)
  808.                         throws IllegalAccessException, IllegalArgumentException, InvocationTargetException,
  809.                         InterruptedException, SmackException, IOException, XMPPException {
  810.             switch (parameterType) {
  811.             case singleConnectedConnection:
  812.             case collectionOfConnections:
  813.             case parameterListOfConnections:
  814.                 final boolean collectionOfConnections = parameterType == TestMethodParameterType.collectionOfConnections;

  815.                 final int connectionCount;
  816.                 if (collectionOfConnections) {
  817.                     connectionCount = smackIntegrationTestAnnotation.connectionCount();
  818.                     if (connectionCount < 1) {
  819.                         throw new IllegalArgumentException(testMethod + " is annotated to use less than one connection ('"
  820.                                         + connectionCount + ')');
  821.                     }
  822.                 } else {
  823.                     connectionCount = testMethod.getParameterCount();
  824.                 }

  825.                 List<? extends AbstractXMPPConnection> connections = connectionManager.constructConnectedConnections(
  826.                                 connectionDescriptor, connectionCount);

  827.                 if (collectionOfConnections) {
  828.                     testMethod.invoke(test, connections);
  829.                 } else {
  830.                     Object[] connectionsArray = new Object[connectionCount];
  831.                     for (int i = 0; i < connectionsArray.length; i++) {
  832.                         connectionsArray[i] = connections.remove(0);
  833.                     }
  834.                     testMethod.invoke(test, connectionsArray);
  835.                 }

  836.                 connectionManager.recycle(connections);
  837.                 break;
  838.             case unconnectedConnectionSource:
  839.                 AbstractSmackLowLevelIntegrationTest.UnconnectedConnectionSource source = () -> {
  840.                     try {
  841.                         return environment.connectionManager.constructConnection(connectionDescriptor);
  842.                     } catch (NoResponseException | XMPPErrorException | NotConnectedException
  843.                                     | InterruptedException e) {
  844.                         // TODO: Ideally we would wrap the exceptions in an unchecked exceptions, catch those unchecked
  845.                         // exceptions below and throw the wrapped checked exception.
  846.                         throw new RuntimeException(e);
  847.                     }
  848.                 };
  849.                 testMethod.invoke(test, source);
  850.                 break;
  851.             case noParamSpecificLowLevel:
  852.                 testMethod.invoke(test);
  853.                 break;
  854.             }
  855.         }

  856.         @Override
  857.         public String toString() {
  858.             return testMethod.toString();
  859.         }
  860.     }

  861.     enum TestMethodParameterType {
  862.         /**
  863.          * testMethod(Connection connection)
  864.          */
  865.         singleConnectedConnection,

  866.         /**
  867.          * testMethod(Collection&lt;Connection&gt;)
  868.          * <p> It can also be a subclass of collection like List. In fact, the type of the parameter being List is expected to be the common case.
  869.          */
  870.         collectionOfConnections,

  871.         /**
  872.          * testMethod(Connection a, Connection b, Connection c)
  873.          */
  874.         parameterListOfConnections,

  875.         /**
  876.          * testMethod(UnconnectedConnectionSource unconnectedConnectionSource)
  877.          */
  878.         unconnectedConnectionSource,

  879.         /**
  880.          * A no-parameter method of a {@link AbstractSmackSpecificLowLevelIntegrationTest}.
  881.          */
  882.         noParamSpecificLowLevel,
  883.     };

  884.     static TestMethodParameterType determineTestMethodParameterType(Method testMethod) {
  885.         return determineTestMethodParameterType(testMethod, AbstractXMPPConnection.class);
  886.     }

  887.     static TestMethodParameterType determineTestMethodParameterType(Method testMethod, Class<? extends AbstractXMPPConnection> connectionClass) {
  888.         Class<?>[] parameterTypes = testMethod.getParameterTypes();
  889.         if (parameterTypes.length == 0) {
  890.             if (AbstractSmackSpecificLowLevelIntegrationTest.class.isAssignableFrom(testMethod.getDeclaringClass())) {
  891.                 return TestMethodParameterType.noParamSpecificLowLevel;
  892.             }
  893.             return null;
  894.         }

  895.         if (parameterTypes.length > 1) {
  896.             // If there are more parameters, then all must be assignable from the connection class.
  897.             for (Class<?> parameterType : parameterTypes) {
  898.                 if (!connectionClass.isAssignableFrom(parameterType)) {
  899.                     return null;
  900.                 }
  901.             }

  902.             return TestMethodParameterType.parameterListOfConnections;
  903.         }

  904.         // This method has exactly a single parameter.
  905.         Class<?> soleParameter = parameterTypes[0];

  906.         if (Collection.class.isAssignableFrom(soleParameter)) {
  907.             // The sole parameter is assignable from collection, which means that it is a parameterized generic type.
  908.             ParameterizedType soleParameterizedType = (ParameterizedType) testMethod.getGenericParameterTypes()[0];
  909.             Type[] actualTypeArguments = soleParameterizedType.getActualTypeArguments();
  910.             if (actualTypeArguments.length != 1) {
  911.                 // The parameter list of the Collection has more than one type.
  912.                 return null;
  913.             }

  914.             Type soleActualTypeArgument = actualTypeArguments[0];
  915.             if (!(soleActualTypeArgument instanceof Class<?>)) {
  916.                 return null;
  917.             }

  918.             Class<?> soleActualTypeArgumentAsClass = (Class<?>) soleActualTypeArgument;
  919.             if (!connectionClass.isAssignableFrom(soleActualTypeArgumentAsClass)) {
  920.                 return null;
  921.             }

  922.             return TestMethodParameterType.collectionOfConnections;
  923.         } else if (connectionClass.isAssignableFrom(soleParameter)) {
  924.             return TestMethodParameterType.singleConnectedConnection;
  925.         } else if (AbstractSmackLowLevelIntegrationTest.UnconnectedConnectionSource.class.isAssignableFrom(soleParameter)) {
  926.             return TestMethodParameterType.unconnectedConnectionSource;
  927.         }

  928.         return null;
  929.     }

  930. }