Merge pull request #4586 from esl/tomasz/MIM-2364-rerun-flaky-tests

[CI] Enable "Rerun Failed Tests" for Small Tests

Author: chrzaszcz

.circleci/template.yml

@@ -326,9 +326,8 @@ commands:
     steps:
       - run:
           name: Coverage
-          when: on_success
+          when: always
           command: |
-            echo "Success!"
             ./rebar3 codecov analyze
             tools/circle-upload-codecov.sh
   run_small_tests:
@@ -342,6 +341,18 @@ commands:
           name: Run Small Tests
           command: |
             tools/test.sh -p small_tests -s true -e true
+      - run:
+          when: always
+          name: Copy small test results for Insights
+          command: |
+            LATEST_CT_RUN=$(ls -td _build/test/logs/ct_run.* 2>/dev/null | head -1)
+            if [ -n "$LATEST_CT_RUN" ] && [ -f "$LATEST_CT_RUN/junit_report.xml" ]; then
+              mkdir -p small_tests_test_results
+              cp "$LATEST_CT_RUN/junit_report.xml" small_tests_test_results/junit_report.xml
+            fi
+      - store_test_results:
+          when: always
+          path: small_tests_test_results
   run_docker_smoke_test:
     steps:
       - checkout
@@ -648,10 +659,16 @@ jobs:
                 when: always
                 name: Copy test results for Insights
                 command: |
-                  cp big_tests/ct_report/*/junit_report.xml .
+                  LATEST_BIG_CT_RUN=$(ls -td big_tests/ct_report/ct_run.* 2>/dev/null | head -1)
+                  if [ -n "$LATEST_BIG_CT_RUN" ] && [ -f "$LATEST_BIG_CT_RUN/junit_report.xml" ]; then
+                    mkdir -p big_tests_test_results
+                    cp "$LATEST_BIG_CT_RUN/junit_report.xml" big_tests_test_results/junit_report.xml
+                  else
+                    echo "No big test junit_report.xml found"
+                  fi
             - store_test_results:
                 when: always
-                path: junit_report.xml
+                path: big_tests_test_results
       - run_coverage_analysis
       - run:
           name: Build Failed - Logs

Makefile

@@ -20,7 +20,7 @@ clean:
 # REBAR_CT_EXTRA_ARGS comes from a test runner
 ct:
 	@(if [ "$(SUITE)" ]; \
-		then $(RUN) $(REBAR) ct --dir test --suite $(SUITE) ; \
+		then $(RUN) $(REBAR) ct --dir test --suite $(SUITE) $(REBAR_CT_EXTRA_ARGS); \
 		else $(RUN) $(REBAR) ct $(REBAR_CT_EXTRA_ARGS); fi)
 
 eunit:

rebar.config

@@ -180,6 +180,7 @@
                                 {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]}]},
                         {deps, [{proper, "1.5.0"}, {meck, "1.0.0"}, {wait_helper, "0.2.1"}]}]},
              {test,    [{extra_src_dirs, [{"test", [{recursive, true}]}]},
+                        {ct_opts, [{ct_hooks, [cth_surefire]}]},
                         {deps, [{proper, "1.5.0"}, {meck, "1.0.0"}, {wait_helper, "0.2.1"}]}]}
             ]}.
 

tools/circle-publish-github-comment.sh

@@ -85,12 +85,10 @@ function last_ct_run_name
 }
 
 
-function small_suite_path
+function last_small_ct_run_name
 {
     if [ -d _build/test/logs ]; then
-        cd _build/test/logs
-        ls -t -1 ct_run.mongooseim@localhost.*/lib.mongooseim.logs/run.*/suite.log.html
-        cd ../../..
+        ls -1 -t _build/test/logs/ | grep ct_run | head -n1
     fi
 }
 
@@ -106,8 +104,8 @@ function ct_small_url
 {
     local CT_REPORTS=$(ct_reports_dir)
     local SMALL_TESTS_URL="$(archive_reader_url small ${CT_REPORTS})"
-    local SUFFIX=$(small_suite_path)
-    echo "$SMALL_TESTS_URL/$SUFFIX"
+    local RUN_PART=$(echo "$(last_small_ct_run_name)" | sed "s/@/%40/g")
+    echo "$SMALL_TESTS_URL/$RUN_PART/index.html"
 }
 
 function reports_url

tools/test.sh

@@ -78,13 +78,51 @@ choose_newest_directory() {
   fi
 }
 
+circleci_tests_available() {
+  command -v circleci >/dev/null 2>&1 && [[ "$CIRCLECI" == "true" ]]
+}
+
+select_circleci_suites() {
+  local glob_pattern="$1"
+  local output_file="$2"
+
+  rm -f "$output_file"
+  # Use xargs to normalize input to newlines, then sed to process
+  # This follows CircleCI docs recommendation to use xargs for input handling
+  if ! circleci tests glob "$glob_pattern" | \
+      circleci tests run --command="xargs -n1 echo | sed 's|.*/||; s|\.erl$||' >>$output_file" --verbose
+  then
+    echo "circleci tests run failed for pattern $glob_pattern"
+    return 1
+  fi
+
+  if [ ! -s "$output_file" ]; then
+    echo "circleci tests run returned no suites for pattern $glob_pattern"
+    return 1
+  fi
+}
+
+maybe_select_small_test_suites() {
+  if circleci_tests_available && select_circleci_suites "test/**/*_SUITE.erl" selected_small_suites; then
+    if [ -s selected_small_suites ]; then
+      # Convert newline-separated suites to comma-separated list for rebar3
+      SELECTED_SUITES=$(cat selected_small_suites | tr '\n' ',' | sed 's/,$//')
+      echo "Selected small test suites: $SELECTED_SUITES"
+      export SUITE="$SELECTED_SUITES"
+    fi
+  else
+    echo "CircleCI did not return specific small-test suites to rerun; executing default list"
+  fi
+}
+
 run_small_tests() {
   tools/print-dots.sh start
   tools/print-dots.sh monitor $$
   if [ "$COVER_ENABLED" = true ]; then
     REBAR_CT_EXTRA_ARGS=" -c $REBAR_CT_EXTRA_ARGS "
   fi
   export REBAR_CT_EXTRA_ARGS="$REBAR_CT_EXTRA_ARGS"
+  maybe_select_small_test_suites
   make ct
   RESULT="$?"
   tools/print-dots.sh stop
@@ -159,11 +197,15 @@ run_test_preset() {
 }
 
 maybe_select_suites() {
-    if command -v circleci >/dev/null 2>&1 && [[ "$CIRCLECI" == "true" ]]; then
-        circleci tests glob tests/*_SUITE.erl | \
-            circleci tests run --command=">selected_suites xargs -d' ' -I {} basename {} .erl"
-        escript ../tools/select_suites_to_run.erl $TESTSPEC $(<selected_suites)
+  if circleci_tests_available && select_circleci_suites "tests/*_SUITE.erl" selected_suites; then
+    if [ -s selected_suites ]; then
+      escript ../tools/select_suites_to_run.erl $TESTSPEC $(<selected_suites)
+    else
+      echo "CircleCI returned an empty suite list for big tests; running defaults"
     fi
+  else
+    echo "CircleCI did not provide failed big-test suites; running defaults"
+  fi
 }
 
 print_running_nodes() {