https://github.com/matrix-org/dendrite fork for cross-compile for ARM
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

105 lines
3.2 KiB

  1. #! /bin/bash
  2. #
  3. # Parses a results.tap file from SyTest output and a file containing test names (a test whitelist)
  4. # and checks whether a test name that exists in the whitelist (that should pass), failed or not.
  5. #
  6. # An optional blacklist file can be added, also containing test names, where if a test name is
  7. # present, the script will not error even if the test is in the whitelist file and failed
  8. #
  9. # For each of these files, lines starting with '#' are ignored.
  10. #
  11. # Usage ./show-expected-fail-tests.sh results.tap whitelist [blacklist]
  12. results_file=$1
  13. whitelist_file=$2
  14. blacklist_file=$3
  15. fail_build=0
  16. if [ $# -lt 2 ]; then
  17. echo "Usage: $0 results.tap whitelist [blacklist]"
  18. exit 1
  19. fi
  20. if [ ! -f "$results_file" ]; then
  21. echo "ERROR: Specified results file '${results_file}' doesn't exist."
  22. fail_build=1
  23. fi
  24. if [ ! -f "$whitelist_file" ]; then
  25. echo "ERROR: Specified test whitelist '${whitelist_file}' doesn't exist."
  26. fail_build=1
  27. fi
  28. blacklisted_tests=()
  29. # Check if a blacklist file was provided
  30. if [ $# -eq 3 ]; then
  31. # Read test blacklist file
  32. if [ ! -f "$blacklist_file" ]; then
  33. echo "ERROR: Specified test blacklist file '${blacklist_file}' doesn't exist."
  34. fail_build=1
  35. fi
  36. # Read each line, ignoring those that start with '#'
  37. blacklisted_tests=""
  38. search_non_comments=$(grep -v '^#' ${blacklist_file})
  39. while read -r line ; do
  40. # Record the blacklisted test name
  41. blacklisted_tests+=("${line}")
  42. done <<< "${search_non_comments}" # This allows us to edit blacklisted_tests in the while loop
  43. fi
  44. [ "$fail_build" = 0 ] || exit 1
  45. passed_but_expected_fail=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//')
  46. tests_to_add=""
  47. already_in_whitelist=""
  48. while read -r test_name; do
  49. # Ignore empty lines
  50. [ "${test_name}" = "" ] && continue
  51. grep "^${test_name}" "${whitelist_file}" > /dev/null 2>&1
  52. if [ "$?" != "0" ]; then
  53. # Check if this test name is blacklisted
  54. if printf '%s\n' "${blacklisted_tests[@]}" | grep -q -P "^${test_name}$"; then
  55. # Don't notify about this test
  56. continue
  57. fi
  58. # Append this test_name to the existing list
  59. tests_to_add="${tests_to_add}${test_name}\n"
  60. fail_build=1
  61. else
  62. already_in_whitelist="${already_in_whitelist}${test_name}\n"
  63. fi
  64. done <<< "${passed_but_expected_fail}"
  65. # TODO: Check that the same test doesn't exist in both the whitelist and blacklist
  66. # TODO: Check that the same test doesn't appear twice in the whitelist|blacklist
  67. # Trim test output strings
  68. tests_to_add=$(IFS=$'\n' echo "${tests_to_add[*]%%'\n'}")
  69. already_in_whitelist=$(IFS=$'\n' echo "${already_in_whitelist[*]%%'\n'}")
  70. # Format output with markdown for buildkite annotation rendering purposes
  71. if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then
  72. echo "### 📜 SyTest Whitelist Maintenance"
  73. fi
  74. if [ -n "${tests_to_add}" ]; then
  75. echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:"
  76. echo "\`\`\`"
  77. echo -e "${tests_to_add}"
  78. echo "\`\`\`"
  79. fi
  80. if [ -n "${already_in_whitelist}" ]; then
  81. echo "**WARN**: Tests in the whitelist still marked as **expected fail**:"
  82. echo "\`\`\`"
  83. echo -e "${already_in_whitelist}"
  84. echo "\`\`\`"
  85. fi
  86. exit ${fail_build}