From 25876f22e2b8f022f01297bc434bff3ff35d3892 Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Tue, 18 Aug 2020 15:54:54 +0200
Subject: [PATCH 1/9] MVP
---
ci/common.yml | 3 +++
common.sh | 35 +++++++++++++++++++++++++++++++++++
testallthethings.sh | 1 +
3 files changed, 39 insertions(+)
diff --git a/ci/common.yml b/ci/common.yml
index dddd9a6..270f293 100644
--- a/ci/common.yml
+++ b/ci/common.yml
@@ -8,6 +8,9 @@
echo "${IMAGECI_PRIV_KEY}" > imageci.pem
chmod 400 imageci.pem
allow_failure: true
+ artifacts:
+ reports:
+ junit: _junit/result.xml
.get_latest_image: &get_latest_image
- |
diff --git a/common.sh b/common.sh
index aa48746..9ddf4f2 100644
--- a/common.sh
+++ b/common.sh
@@ -142,3 +142,38 @@ function showFlavorResults() {
done
}
+
+function createJUnit() {
+ mkdir "_junit"
+ XML="_junit/result.xml"
+
+ echo "<?xml version='1.0' encoding='UTF-8'?>" > $XML
+ echo "<testsuites>" >> $XML
+
+ local flavors="$(echo "${!RESULTS[@]}" | tr ' ' '\n' | sort)"
+ for flavor in $flavors; do
+ local result_num="`echo "${RESULTS[${flavor}]}" | cut -d';' -f1`"
+ local comment="`echo "${RESULTS[${flavor}]}" | cut -d';' -f2-`"
+
+ echo "<testsuite name='test' tests=''>" >> $XML
+
+ echo "<testcase name='${flavor}' classname='${flavor}' time='1'>" >> $XML
+
+ if [[ $result_num -eq $RESULT_UNTESTED ]]; then
+ echo "<skipped message='Not tested' />" >> $XML
+ elif [[ $result_num -eq $RESULT_FAIL ]]; then
+
+ echo "<failure>" >> $XML
+ # Print comments indented, one per line
+ while IFS= read -r LINE; do
+ [[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
+ done <<< "$(echo "${comment}" | tr ";" "\n")"
+
+ echo "</failure>" >> $XML
+ fi
+
+ echo '</testcase></testsuite>' >> $XML
+ done
+
+ echo "</testsuites>" >> $XML
+}
diff --git a/testallthethings.sh b/testallthethings.sh
index 60f97ac..3c2ec8c 100755
--- a/testallthethings.sh
+++ b/testallthethings.sh
@@ -298,5 +298,6 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
done
showFlavorResults "${IMAGE}"
+createJUnit "${IMAGE}"
exit $GLOBAL_RETURN
--
GitLab
From d095d8c57833a04b79c3593c584b9ab8af0018e4 Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Wed, 19 Aug 2020 18:04:24 +0200
Subject: [PATCH 2/9] New way of defining tests
---
common.sh | 174 +++++++++++++++++++++++++++++++++++++++++---
testallthethings.sh | 44 ++++++++++-
2 files changed, 205 insertions(+), 13 deletions(-)
diff --git a/common.sh b/common.sh
index 9ddf4f2..01f1e14 100644
--- a/common.sh
+++ b/common.sh
@@ -5,8 +5,15 @@ declare -r RESULT_PASS=1
declare -r RESULT_SOFTPASS=2
declare -r RESULT_FAIL=3
+declare -r TEST_SKIPPED=0
+declare -r TEST_RUNNING=1
+declare -r TEST_ERROR=2
+declare -r TEST_FAIL=3
+declare -r TEST_PASS=4
+
# Test results
declare -A RESULTS
+declare TESTS=()
function t_Log {
echo -e "[+] `date` -> $*"
@@ -143,36 +150,179 @@ function showFlavorResults() {
done
}
-function createJUnit() {
+# Test format
+# flavor;name;starttime;endtime;status;comments
+
+function testUpdate {
+ local FLAVOR="$1"
+ local NAME="$2"
+ local STARTTIME="$3"
+ local ENDTIME="$4"
+ local STATUS="$5"
+ local COMMENTS="$6"
+
+ for i in ${!TESTS[@]}; do
+ local flavor="`echo "${TESTS[$i]}" | cut -d';' -f1`"
+ local name="`echo "${TESTS[$i]}" | cut -d';' -f2`"
+
+ if [[ "$FLAVOR" == "$flavor" && "$NAME" == "$name" ]]; then
+ local start="`echo "${TESTS[$i]}" | cut -d';' -f3`"
+ local end="`echo "${TESTS[$i]}" | cut -d';' -f4`"
+ local status="`echo "${TESTS[$i]}" | cut -d';' -f5`"
+ local comments="`echo "${TESTS[$i]}" | cut -d';' -f6-`"
+
+ [[ "$STARTTIME" == "undef" ]] && STARTTIME="$start"
+ [[ "$ENDTIME" == "undef" ]] && ENDTIME="$end"
+ [[ "$STATUS" == "undef" ]] && STATUS="$status"
+ [[ "$COMMENTS" == "undef" ]] && COMMENTS="$comments"
+
+ TESTS[$i]="$FLAVOR;$NAME;$STARTTIME;$ENDTIME;$STATUS;$COMMENTS"
+ return
+ fi
+ done
+
+ [[ "$STARTTIME" == "undef" ]] && STARTTIME=""
+ [[ "$ENDTIME" == "undef" ]] && ENDTIME=""
+ [[ "$STATUS" == "undef" ]] && STATUS=""
+ [[ "$COMMENTS" == "undef" ]] && COMMENTS=""
+
+ TESTS+=("$FLAVOR;$NAME;$STARTTIME;$ENDTIME;$STATUS;$COMMENTS")
+}
+
+function testGet {
+ local FLAVOR="$1"
+ local NAME="$2"
+
+ for i in ${!TESTS[@]}; do
+ local flavor="`echo "${TESTS[$i]}" | cut -d';' -f1`"
+ local name="`echo "${TESTS[$i]}" | cut -d';' -f2`"
+
+ if [[ "$FLAVOR" == "$flavor" && "$NAME" == "$name" ]]; then
+ echo "${TESTS[$i]}"
+ fi
+ done
+}
+
+function testGetResult {
+ local FLAVOR="$1"
+ local NAME="$2"
+
+ echo "`testGet "$FLAVOR" "$NAME"`" | cut -d';' -f5
+}
+
+function testCreate {
+ local FLAVOR="$1"
+ local NAME="$2"
+
+ testUpdate "$FLAVOR" "$NAME" undef undef $TEST_SKIPPED undef
+}
+
+function testStart {
+ local FLAVOR=$1
+ local NAME=$2
+ local START=`date +%s`
+
+ testUpdate "$FLAVOR" "$NAME" "$START" undef $TEST_RUNNING undef
+}
+
+function testEnd {
+ local FLAVOR=$1
+ local NAME=$2
+ local STATUS=$3
+ local END=`date +%s`
+
+ testUpdate "$FLAVOR" "$NAME" undef "$END" "$STATUS" undef
+}
+
+function testAddComments {
+ local FLAVOR=$1
+ shift
+ local NAME=$1
+ shift
+ local COMMENTS="${*//;}" # remove ; from the comment, reserved char
+
+ local result="`testGet "$FLAVOR" "$NAME"`"
+
+ local current_comment="`echo "$result" | cut -d';' -f6-`"
+ # If we already had a comment, concatenate the new one with a ;
+ [[ ! -z "$current_comment" ]] && COMMENTS="$current_comment;$COMMENTS"
+
+ testUpdate "$FLAVOR" "$NAME" undef undef undef "$COMMENTS"
+}
+
+function testShowResults {
+ t_Log "Finished tests of image '\e[1m${1}\e[0m'."
+ t_Log "Here are the results:"
+
+ for i in ${!TESTS[@]}; do
+ local flavor="`echo "${TESTS[$i]}" | cut -d';' -f1`"
+ local name="`echo "${TESTS[$i]}" | cut -d';' -f2`"
+ local start="`echo "${TESTS[$i]}" | cut -d';' -f3`"
+ local end="`echo "${TESTS[$i]}" | cut -d';' -f4`"
+ local status="`echo "${TESTS[$i]}" | cut -d';' -f5`"
+ local comments="`echo "${TESTS[$i]}" | cut -d';' -f6-`"
+
+ case $status in
+ $TEST_SKIPPED)
+ result="SKIPPED" ;;
+ $TEST_PASS)
+ result="\e[1m\e[32mPASS\e[0m" ;;
+ $TEST_ERROR)
+ result="\e[31mERROR\e[0m" ;;
+ $TEST_FAIL)
+ result="\e[1m\e[31mFAIL\e[0m" ;;
+ esac
+
+ t_Log " ${flavor} ${name} -> ${result}"
+
+ # Print comments indented, one per line
+ while IFS= read -r LINE; do
+ [[ ! -z "$LINE" ]] && t_Log " $LINE"
+ done <<< "$(echo "${comments}" | tr ";" "\n")"
+
+ done
+}
+
+function testCreateJUnit() {
mkdir "_junit"
- XML="_junit/result.xml"
+ local XML="_junit/result.xml"
echo "<?xml version='1.0' encoding='UTF-8'?>" > $XML
echo "<testsuites>" >> $XML
- local flavors="$(echo "${!RESULTS[@]}" | tr ' ' '\n' | sort)"
- for flavor in $flavors; do
- local result_num="`echo "${RESULTS[${flavor}]}" | cut -d';' -f1`"
- local comment="`echo "${RESULTS[${flavor}]}" | cut -d';' -f2-`"
+ for i in ${!TESTS[@]}; do
+ local flavor="`echo "${TESTS[$i]}" | cut -d';' -f1`"
+ local name="`echo "${TESTS[$i]}" | cut -d';' -f2`"
+ local start="`echo "${TESTS[$i]}" | cut -d';' -f3`"
+ local end="`echo "${TESTS[$i]}" | cut -d';' -f4`"
+ local status="`echo "${TESTS[$i]}" | cut -d';' -f5`"
+ local comments="`echo "${TESTS[$i]}" | cut -d';' -f6-`"
- echo "<testsuite name='test' tests=''>" >> $XML
+ local time=$(($end-$start))
- echo "<testcase name='${flavor}' classname='${flavor}' time='1'>" >> $XML
+ echo "<testsuite name='test' tests=''>" >> $XML
+ echo "<testcase name='${flavor}' classname='${name}' time='${time}'>" >> $XML
- if [[ $result_num -eq $RESULT_UNTESTED ]]; then
+ if [[ $result_num -eq $TEST_SKIPPED ]]; then
echo "<skipped message='Not tested' />" >> $XML
- elif [[ $result_num -eq $RESULT_FAIL ]]; then
-
+ elif [[ $result_num -eq $TEST_ERROR ]]; then
+ echo "<error>" >> $XML
+ # Print comments indented, one per line
+ while IFS= read -r LINE; do
+ [[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
+ done <<< "$(echo "${comment}" | tr ";" "\n")"
+ echo "</error>" >> $XML
+ elif [[ $result_num -eq $TEST_FAIL ]]; then
echo "<failure>" >> $XML
# Print comments indented, one per line
while IFS= read -r LINE; do
[[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
done <<< "$(echo "${comment}" | tr ";" "\n")"
-
echo "</failure>" >> $XML
fi
echo '</testcase></testsuite>' >> $XML
+
done
echo "</testsuites>" >> $XML
diff --git a/testallthethings.sh b/testallthethings.sh
index 3c2ec8c..2baabb1 100755
--- a/testallthethings.sh
+++ b/testallthethings.sh
@@ -128,6 +128,16 @@ fi
t_Log "Openstack flavors to test:"
for f in "${FLAVORS[@]}"; do
t_Log " ${f}"
+
+ # Create entries for the tests we're going to run
+ testCreate "${f}" "create"
+ testCreate "${f}" "ssh_access"
+ if [[ "$PUPPET" == "true" ]]; then
+ testCreate "${f}" "puppet_tests"
+ else
+ testCreate "${f}" "upstream_centos_tests"
+ testCreate "${f}" "cern_centos_tests"
+ fi
done
# Poor man's FIFO queue
@@ -145,6 +155,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
# If we've already retried too much, give up
if [[ $COUNT -gt $MIN_RETRY_COUNT ]]; then
+ testAddComments "${flavor}" "create" "All machines of this type were busy"
setFlavorResult "$flavor" "$RESULT_UNTESTED" "All machines of this type were busy"
continue
fi
@@ -167,17 +178,22 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
sleep $DELAY
cmd="ai-bs --nova-sshkey imageci --landb-responsible imageci --nova-parameter centos_test_cleanup=true --nova-flavor '${flavor}' --nova-image '${IMAGE}' --foreman-hostgroup playground/imageci '${p_name}'"
t_BoldGreen "${cmd}"
+ testStart "${flavor}" "create"
cmd_output=$(eval ${cmd} 2>&1)
else
cmd="openstack server create --key-name imageci --property landb-responsible=imageci --property centos_test_cleanup=true --flavor '${flavor}' --image '${IMAGE}' '${p_name}'"
t_BoldGreen "${cmd}"
+ testStart "${flavor}" "create"
cmd_output=$(eval ${cmd} -c id --format value 2>&1)
fi
if [[ $? -ne 0 ]]; then
# Unable to create the server, remove it from the list
t_Log "Unable to create server of flavor ${flavor}"
+ testEnd "${flavor}" "create" $TEST_ERROR
+ testAddComments "${flavor}" "create" "Unable to create server"
setFlavorResult "$flavor" "$RESULT_UNTESTED" "Unable to create server"
# Add the error to the comments
+ testAddComments "${flavor}" "create" "${cmd_output}"
appendFlavorComment "$flavor" "${cmd_output}"
continue
fi
@@ -186,6 +202,8 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
waitFor 30 60 isServerCreated ${p_name}
if [[ $? -eq 1 ]]; then
t_Log "Timed out while creating a server of flavor ${flavor}"
+ testEnd "${flavor}" "create" $TEST_ERROR
+ testAddComments "${flavor}" "create" "Timed out while creating server"
setFlavorResult "$flavor" "$RESULT_UNTESTED" "Timed out while creating server"
deleteServer "${p_name}" "${flavor}"
continue
@@ -197,6 +215,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
if [[ $status == 'ERROR' ]]; then
echo "Fault message: $error"
+ testEnd "${flavor}" "create" $TEST_ERROR
openstack server list
if [[ $error =~ "No valid host was found" ]]; then
# Is this the first time we've tried this flavor?
@@ -222,6 +241,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
else
# No machine of the type currently exists
t_Log " No other machine found, maybe none are available with this flavor?"
+ testAddComments "${flavor}" "create" "No machines of this type found"
setFlavorResult "$flavor" "$RESULT_UNTESTED" "No machines of this type found"
fi
else
@@ -235,13 +255,16 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
else
t_CheckExitStatus 1
t_Log "Unexpected error while creating machine: $error"
+ testAddComments "${flavor}" "create" "$error"
setFlavorResult "$flavor" "$RESULT_UNTESTED" "$error"
fi
else
+ testEnd "${flavor}" "create" $TEST_PASS
t_Log "Machine created successfully!"
openstack server show ${p_name}
t_Log "Trying to SSH into the machine"
+ testStart "${flavor}" "ssh_access"
# Wait for the machine to really be up. Retry every 30 seconds (+ 10 seconds SSH timeout), up to 15 times => wait 10min
# For Puppet-managed machines, wait 40 min. (Puppet and distro-sync have to run too)
[[ "$PUPPET" == "true" ]] && wait=80 || wait=15
@@ -251,40 +274,58 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
GLOBAL_RETURN=$((GLOBAL_RETURN+g))
t_CheckExitStatus $g
if [[ $g -ne 0 ]]; then
+ testEnd "${flavor}" "ssh_access" $TEST_FAIL
+ testAddComments "${flavor}" "ssh_access" "Unable to SSH into the machine"
setFlavorResult "$flavor" "$RESULT_FAIL" "Unable to SSH into the machine"
deleteServer "${p_name}" "${flavor}"
continue
fi
+ testEnd "${flavor}" "ssh_access" $TEST_PASS
# Upstream tests will fail too much for Puppet-managed machines, so instead for those we'll
# check if Puppet ran cleanly.
if [[ "$PUPPET" == "true" ]]; then
t_Log "We're in! Running Puppet tests"
+ testStart "${flavor}" "puppet_tests"
runOnServer "${p_name}" "PUPPET_TEST_TOKEN="${PUPPET_TEST_TOKEN}" bash -s" < ./puppettests.sh
g=$?
t_CheckExitStatus $g
if [[ $g -ne 0 ]]; then
+ testEnd "${flavor}" "puppet_tests" $TEST_FAIL
+ testAddComments "${flavor}" "puppet_tests" "Puppet tests failed"
setFlavorResult "$flavor" "$RESULT_SOFTPASS" "Puppet tests failed"
+ else
+ testEnd "${flavor}" "puppet_tests" $TEST_PASS
fi
else
t_Log "We're in! Running upstream tests"
# Find the OS we're running, and the commit hash of the latest passing test. Tested machines may not have external connectivity.
centos_ver=$(runOnServer "${p_name}" "lsb_release -rs | cut -f 1 -d '.'")
PASSING=$(curl -s https://ci.centos.org/job/CentOS-Core-QA-t_functional-c${centos_ver}-64/lastSuccessfulBuild/api/json | jq -r '.actions[] | select(.lastBuiltRevision) | .lastBuiltRevision.SHA1')
+ testStart "${flavor}" "upstream_centos_tests"
# Now run the actual tests
runOnServer "${p_name}" "bash -s ${PASSING}" < ./upstreamcentostests.sh
g=$?
t_CheckExitStatus $g
if [[ $g -ne 0 ]]; then
+ testEnd "${flavor}" "upstream_centos_tests" $TEST_FAIL
+ testAddComments "${flavor}" "upstream_centos_tests" "Upstream tests failed"
setFlavorResult "$flavor" "$RESULT_SOFTPASS" "Upstream tests failed"
+ else
+ testEnd "${flavor}" "upstream_centos_tests" $TEST_PASS
fi
t_Log "Now run our own tests"
+ testStart "${flavor}" "cern_centos_tests"
runOnServer "${p_name}" "IMAGECI_USER='$IMAGECI_USER' IMAGECI_PWD='$IMAGECI_PWD' bash -s" < ./cerncentostests.sh
g=$?
t_CheckExitStatus $g
if [[ $g -ne 0 ]]; then
+ testEnd "${flavor}" "cern_centos_tests" $TEST_FAIL
+ testAddComments "${flavor}" "cern_centos_tests" "CERN tests failed"
setFlavorResult "$flavor" "$RESULT_SOFTPASS" "CERN tests failed"
+ else
+ testEnd "${flavor}" "cern_centos_tests" $TEST_PASS
fi
fi
@@ -298,6 +339,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
done
showFlavorResults "${IMAGE}"
-createJUnit "${IMAGE}"
+testShowResults
+testCreateJUnit
exit $GLOBAL_RETURN
--
GitLab
From 61652518b2705697422f4304678c3ed371feb40c Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Wed, 19 Aug 2020 18:52:41 +0200
Subject: [PATCH 3/9] Bugfixes
---
ci/common.yml | 2 ++
common.sh | 10 +++++-----
testallthethings.sh | 2 +-
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/ci/common.yml b/ci/common.yml
index 270f293..1f09b37 100644
--- a/ci/common.yml
+++ b/ci/common.yml
@@ -9,6 +9,8 @@
chmod 400 imageci.pem
allow_failure: true
artifacts:
+ paths:
+ - _junit/result.xml
reports:
junit: _junit/result.xml
diff --git a/common.sh b/common.sh
index 01f1e14..7b39faa 100644
--- a/common.sh
+++ b/common.sh
@@ -303,21 +303,21 @@ function testCreateJUnit() {
echo "<testsuite name='test' tests=''>" >> $XML
echo "<testcase name='${flavor}' classname='${name}' time='${time}'>" >> $XML
- if [[ $result_num -eq $TEST_SKIPPED ]]; then
+ if [[ $status -eq $TEST_SKIPPED ]]; then
echo "<skipped message='Not tested' />" >> $XML
- elif [[ $result_num -eq $TEST_ERROR ]]; then
+ elif [[ $status -eq $TEST_ERROR ]]; then
echo "<error>" >> $XML
# Print comments indented, one per line
while IFS= read -r LINE; do
[[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
- done <<< "$(echo "${comment}" | tr ";" "\n")"
+ done <<< "$(echo "${comments}" | tr ";" "\n")"
echo "</error>" >> $XML
- elif [[ $result_num -eq $TEST_FAIL ]]; then
+ elif [[ $status -eq $TEST_FAIL ]]; then
echo "<failure>" >> $XML
# Print comments indented, one per line
while IFS= read -r LINE; do
[[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
- done <<< "$(echo "${comment}" | tr ";" "\n")"
+ done <<< "$(echo "${comments}" | tr ";" "\n")"
echo "</failure>" >> $XML
fi
diff --git a/testallthethings.sh b/testallthethings.sh
index 2baabb1..54efa96 100755
--- a/testallthethings.sh
+++ b/testallthethings.sh
@@ -339,7 +339,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
done
showFlavorResults "${IMAGE}"
-testShowResults
+testShowResults "${IMAGE}"
testCreateJUnit
exit $GLOBAL_RETURN
--
GitLab
From a9399b7c3ae36a87830f5dff44c5682c5fef51d6 Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Thu, 20 Aug 2020 14:18:28 +0200
Subject: [PATCH 4/9] Fix more bugs
---
common.sh | 34 +++++++++++++++++++++++++---------
testallthethings.sh | 1 +
2 files changed, 26 insertions(+), 9 deletions(-)
diff --git a/common.sh b/common.sh
index 7b39faa..fe6c064 100644
--- a/common.sh
+++ b/common.sh
@@ -241,6 +241,7 @@ function testAddComments {
shift
local COMMENTS="${*//;}" # remove ; from the comment, reserved char
+ COMMENTS="$(echo "${COMMENTS}" | tr "\n" ";")"
local result="`testGet "$FLAVOR" "$NAME"`"
local current_comment="`echo "$result" | cut -d';' -f6-`"
@@ -250,6 +251,13 @@ function testAddComments {
testUpdate "$FLAVOR" "$NAME" undef undef undef "$COMMENTS"
}
+function testClearComments {
+ local FLAVOR=$1
+ local NAME=$2
+
+ testUpdate "$FLAVOR" "$NAME" undef undef undef ""
+}
+
function testShowResults {
t_Log "Finished tests of image '\e[1m${1}\e[0m'."
t_Log "Here are the results:"
@@ -286,7 +294,7 @@ function testShowResults {
function testCreateJUnit() {
mkdir "_junit"
local XML="_junit/result.xml"
-
+ set -x
echo "<?xml version='1.0' encoding='UTF-8'?>" > $XML
echo "<testsuites>" >> $XML
@@ -298,32 +306,40 @@ function testCreateJUnit() {
local status="`echo "${TESTS[$i]}" | cut -d';' -f5`"
local comments="`echo "${TESTS[$i]}" | cut -d';' -f6-`"
- local time=$(($end-$start))
+ if [[ -z "$start" || -z "$end" ]]; then
+ local time=0
+ else
+ local time=$(($end-$start))
+ fi
- echo "<testsuite name='test' tests=''>" >> $XML
+ echo '<testsuite name="test" tests="">' >> $XML
echo "<testcase name='${flavor}' classname='${name}' time='${time}'>" >> $XML
if [[ $status -eq $TEST_SKIPPED ]]; then
- echo "<skipped message='Not tested' />" >> $XML
+ echo '<skipped message="Not tested" />' >> $XML
elif [[ $status -eq $TEST_ERROR ]]; then
- echo "<error>" >> $XML
+ echo '<error><![CDATA[' >> $XML
# Print comments indented, one per line
while IFS= read -r LINE; do
[[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
done <<< "$(echo "${comments}" | tr ";" "\n")"
- echo "</error>" >> $XML
+ echo ']]></error>' >> $XML
elif [[ $status -eq $TEST_FAIL ]]; then
- echo "<failure>" >> $XML
+ echo '<failure><![CDATA[' >> $XML
# Print comments indented, one per line
while IFS= read -r LINE; do
[[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
done <<< "$(echo "${comments}" | tr ";" "\n")"
- echo "</failure>" >> $XML
+ echo ']]></failure>' >> $XML
fi
echo '</testcase></testsuite>' >> $XML
done
- echo "</testsuites>" >> $XML
+ echo '</testsuites>' >> $XML
+ set +x
+ echo
+ cat $XML
+ echo
}
diff --git a/testallthethings.sh b/testallthethings.sh
index 54efa96..23083d3 100755
--- a/testallthethings.sh
+++ b/testallthethings.sh
@@ -161,6 +161,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
fi
if [[ ! -z $PREV_TIME ]]; then
+ testClearComments "${flavor}" "create"
# Wait at least $MIN_RETRY_TIME before trying again. Add up to 30 seconds to avoid deadlocks
DELAY=$(( $MIN_RETRY_TIME - (`date +%s` - $PREV_TIME) + ($RANDOM % 30) ))
if [[ $DELAY -gt 0 ]]; then
--
GitLab
From 947c37212d26b9b516da0c1c71c3507b1c64e3a6 Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Thu, 20 Aug 2020 14:43:06 +0200
Subject: [PATCH 5/9] Swap classname/name
---
common.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/common.sh b/common.sh
index fe6c064..0b6013e 100644
--- a/common.sh
+++ b/common.sh
@@ -313,7 +313,7 @@ function testCreateJUnit() {
fi
echo '<testsuite name="test" tests="">' >> $XML
- echo "<testcase name='${flavor}' classname='${name}' time='${time}'>" >> $XML
+ echo "<testcase classname='${flavor}' name='${name}' time='${time}'>" >> $XML
if [[ $status -eq $TEST_SKIPPED ]]; then
echo '<skipped message="Not tested" />' >> $XML
--
GitLab
From dd4858b3a68381170a17f3e1e3399ab1c57c2d94 Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Thu, 20 Aug 2020 14:43:55 +0200
Subject: [PATCH 6/9] Get rid of the newline
---
common.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/common.sh b/common.sh
index 0b6013e..b795c49 100644
--- a/common.sh
+++ b/common.sh
@@ -318,14 +318,14 @@ function testCreateJUnit() {
if [[ $status -eq $TEST_SKIPPED ]]; then
echo '<skipped message="Not tested" />' >> $XML
elif [[ $status -eq $TEST_ERROR ]]; then
- echo '<error><![CDATA[' >> $XML
+ echo -n '<error><![CDATA[' >> $XML
# Print comments indented, one per line
while IFS= read -r LINE; do
[[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
done <<< "$(echo "${comments}" | tr ";" "\n")"
echo ']]></error>' >> $XML
elif [[ $status -eq $TEST_FAIL ]]; then
- echo '<failure><![CDATA[' >> $XML
+ echo -n '<failure><![CDATA[' >> $XML
# Print comments indented, one per line
while IFS= read -r LINE; do
[[ ! -z "$LINE" ]] && echo "$LINE" >> $XML
--
GitLab
From a9d7c8969f20392a39499fbe7d288f6f4dbdc420 Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Thu, 20 Aug 2020 17:21:07 +0200
Subject: [PATCH 7/9] Remove the FlavorResult stuff
---
common.sh | 65 ---------------------------------------------
testallthethings.sh | 32 +++++++++-------------
2 files changed, 12 insertions(+), 85 deletions(-)
diff --git a/common.sh b/common.sh
index b795c49..772aee3 100644
--- a/common.sh
+++ b/common.sh
@@ -1,10 +1,4 @@
# Results constants
-declare -r RESULT_UNKNOWN=-1
-declare -r RESULT_UNTESTED=0
-declare -r RESULT_PASS=1
-declare -r RESULT_SOFTPASS=2
-declare -r RESULT_FAIL=3
-
declare -r TEST_SKIPPED=0
declare -r TEST_RUNNING=1
declare -r TEST_ERROR=2
@@ -12,7 +6,6 @@ declare -r TEST_FAIL=3
declare -r TEST_PASS=4
# Test results
-declare -A RESULTS
declare TESTS=()
function t_Log {
@@ -92,64 +85,6 @@ function isServerUp() {
return 4
}
-function setFlavorResult() {
- local flavor="${1}"
- shift
- local result="${1:-$RESULT_UNKNOWN}"
- shift
- local comment="${*//;}" # remove ; from the comment, reserved char
-
- local current_result="`echo "${RESULTS[${flavor}]}" | cut -d';' -f1`"
- local current_comment="`echo "${RESULTS[${flavor}]}" | cut -d';' -f2-`"
-
- # If we already had a soft pass for this flavor, don't upgrade it to a pass
- [[ $current_result -eq $RESULT_SOFTPASS && $result -eq $RESULT_PASS ]] && result=$current_result
-
- # If we already had a comment, concatenate the new one with a ;
- [[ ! -z "$current_comment" ]] && comment="$current_comment;$comment"
-
- RESULTS[$flavor]="${result};$comment"
-}
-
-function getFlavorResult() {
- echo "${RESULTS[${1}]:-$RESULT_UNKNOWN}" | cut -d';' -f1
-}
-
-function appendFlavorComment() {
- setFlavorResult "$1" "`getFlavorResult "$1"`" "$2"
-}
-
-function showFlavorResults() {
- t_Log "Finished tests of image '\e[1m${1}\e[0m'."
- t_Log "Here are the results:"
-
- local flavors="$(echo "${!RESULTS[@]}" | tr ' ' '\n' | sort)"
- for flavor in $flavors; do
- local result_num="`echo "${RESULTS[${flavor}]}" | cut -d';' -f1`"
- local comment="`echo "${RESULTS[${flavor}]}" | cut -d';' -f2-`"
- local result="UNKNOWN"
-
- case $result_num in
- $RESULT_UNTESTED)
- result="UNTESTED" ;;
- $RESULT_PASS)
- result="\e[1m\e[32mPASS\e[0m" ;;
- $RESULT_SOFTPASS)
- result="\e[32mSOFT PASS\e[0m" ;;
- $RESULT_FAIL)
- result="\e[1m\e[31mFAIL\e[0m" ;;
- esac
-
- t_Log " ${flavor} -> ${result}"
-
- # Print comments indented, one per line
- while IFS= read -r LINE; do
- [[ ! -z "$LINE" ]] && t_Log " $LINE"
- done <<< "$(echo "${comment}" | tr ";" "\n")"
-
- done
-}
-
# Test format
# flavor;name;starttime;endtime;status;comments
diff --git a/testallthethings.sh b/testallthethings.sh
index 23083d3..ace59eb 100755
--- a/testallthethings.sh
+++ b/testallthethings.sh
@@ -77,17 +77,18 @@ function isServerDeleted() {
function deleteServer() {
local p_name="$1"
local flavor="$2"
+ local test="$3"
local delete=true
# Figure out if we should really delete or not
- local result=`getFlavorResult ${flavor}`
- if [[ "$DELETE_FAILURES" == false && $result -ne $RESULT_PASS && $result -ne $RESULT_UNTESTED || $result == "" ]]; then
+ local result=`testGetResult ${flavor} ${test}`
+ if [[ "$DELETE_FAILURES" == false && $result -ne $TEST_PASS && $result -ne $TEST_SKIPPED || $result == "" ]]; then
delete=false
fi
if [[ "$delete" == false ]]; then
# Add a comment to the result
- appendFlavorComment "$flavor" "Machine ${p_name} not deleted, available for debugging"
+ testAddComments "${flavor}" "${test}" "Machine ${p_name} not deleted, available for debugging"
else
t_Log "Deleting ${p_name}"
if [[ "$PUPPET" == "true" ]]; then
@@ -156,7 +157,6 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
# If we've already retried too much, give up
if [[ $COUNT -gt $MIN_RETRY_COUNT ]]; then
testAddComments "${flavor}" "create" "All machines of this type were busy"
- setFlavorResult "$flavor" "$RESULT_UNTESTED" "All machines of this type were busy"
continue
fi
@@ -192,10 +192,8 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
t_Log "Unable to create server of flavor ${flavor}"
testEnd "${flavor}" "create" $TEST_ERROR
testAddComments "${flavor}" "create" "Unable to create server"
- setFlavorResult "$flavor" "$RESULT_UNTESTED" "Unable to create server"
# Add the error to the comments
testAddComments "${flavor}" "create" "${cmd_output}"
- appendFlavorComment "$flavor" "${cmd_output}"
continue
fi
@@ -205,8 +203,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
t_Log "Timed out while creating a server of flavor ${flavor}"
testEnd "${flavor}" "create" $TEST_ERROR
testAddComments "${flavor}" "create" "Timed out while creating server"
- setFlavorResult "$flavor" "$RESULT_UNTESTED" "Timed out while creating server"
- deleteServer "${p_name}" "${flavor}"
+ deleteServer "${p_name}" "${flavor}" "create"
continue
fi
@@ -243,7 +240,6 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
# No machine of the type currently exists
t_Log " No other machine found, maybe none are available with this flavor?"
testAddComments "${flavor}" "create" "No machines of this type found"
- setFlavorResult "$flavor" "$RESULT_UNTESTED" "No machines of this type found"
fi
else
# If we tried it before, that's because at some point we determined it was possible
@@ -257,8 +253,9 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
t_CheckExitStatus 1
t_Log "Unexpected error while creating machine: $error"
testAddComments "${flavor}" "create" "$error"
- setFlavorResult "$flavor" "$RESULT_UNTESTED" "$error"
fi
+
+ deleteServer "${p_name}" "${flavor}" "create"
else
testEnd "${flavor}" "create" $TEST_PASS
t_Log "Machine created successfully!"
@@ -277,8 +274,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
if [[ $g -ne 0 ]]; then
testEnd "${flavor}" "ssh_access" $TEST_FAIL
testAddComments "${flavor}" "ssh_access" "Unable to SSH into the machine"
- setFlavorResult "$flavor" "$RESULT_FAIL" "Unable to SSH into the machine"
- deleteServer "${p_name}" "${flavor}"
+ deleteServer "${p_name}" "${flavor}" "ssh_access"
continue
fi
testEnd "${flavor}" "ssh_access" $TEST_PASS
@@ -294,10 +290,11 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
if [[ $g -ne 0 ]]; then
testEnd "${flavor}" "puppet_tests" $TEST_FAIL
testAddComments "${flavor}" "puppet_tests" "Puppet tests failed"
- setFlavorResult "$flavor" "$RESULT_SOFTPASS" "Puppet tests failed"
else
testEnd "${flavor}" "puppet_tests" $TEST_PASS
fi
+
+ deleteServer "${p_name}" "${flavor}" "puppet_tests"
else
t_Log "We're in! Running upstream tests"
# Find the OS we're running, and the commit hash of the latest passing test. Tested machines may not have external connectivity.
@@ -311,7 +308,6 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
if [[ $g -ne 0 ]]; then
testEnd "${flavor}" "upstream_centos_tests" $TEST_FAIL
testAddComments "${flavor}" "upstream_centos_tests" "Upstream tests failed"
- setFlavorResult "$flavor" "$RESULT_SOFTPASS" "Upstream tests failed"
else
testEnd "${flavor}" "upstream_centos_tests" $TEST_PASS
fi
@@ -324,22 +320,18 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
if [[ $g -ne 0 ]]; then
testEnd "${flavor}" "cern_centos_tests" $TEST_FAIL
testAddComments "${flavor}" "cern_centos_tests" "CERN tests failed"
- setFlavorResult "$flavor" "$RESULT_SOFTPASS" "CERN tests failed"
else
testEnd "${flavor}" "cern_centos_tests" $TEST_PASS
fi
+
+ deleteServer "${p_name}" "${flavor}" "cern_centos_tests"
fi
- setFlavorResult "$flavor" "$RESULT_PASS"
t_Log "All done with ${p_name}!"
fi
- # Delete the server
- deleteServer "${p_name}" "${flavor}"
-
done
-showFlavorResults "${IMAGE}"
testShowResults "${IMAGE}"
testCreateJUnit
--
GitLab
From f9345726962c90e16eea2455b169ce16f543f544 Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Thu, 20 Aug 2020 18:16:03 +0200
Subject: [PATCH 8/9] Replace echo with t_Log
---
testallthethings.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/testallthethings.sh b/testallthethings.sh
index ace59eb..ff8e200 100755
--- a/testallthethings.sh
+++ b/testallthethings.sh
@@ -175,7 +175,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
if [[ "$PUPPET" == "true" ]]; then
# Foreman may deadlock if we try to create too many machines at once, so wait a random bit
DELAY=$(($RANDOM % 10))
- echo sleep $DELAY
+ t_Log "Wait ${DELAY} seconds"
sleep $DELAY
cmd="ai-bs --nova-sshkey imageci --landb-responsible imageci --nova-parameter centos_test_cleanup=true --nova-flavor '${flavor}' --nova-image '${IMAGE}' --foreman-hostgroup playground/imageci '${p_name}'"
t_BoldGreen "${cmd}"
@@ -212,7 +212,7 @@ while [[ ${#TOBETESTED[@]} -gt 0 ]]; do
error=$(echo "$show" | jq -r '.fault.message')
if [[ $status == 'ERROR' ]]; then
- echo "Fault message: $error"
+ t_Log "Fault message: $error"
testEnd "${flavor}" "create" $TEST_ERROR
openstack server list
if [[ $error =~ "No valid host was found" ]]; then
--
GitLab
From eea210235b3d71332bc7685844d184800407c851 Mon Sep 17 00:00:00 2001
From: Alex Iribarren <Alex.Iribarren@cern.ch>
Date: Thu, 20 Aug 2020 19:13:39 +0200
Subject: [PATCH 9/9] Remove debugging stuff
---
common.sh | 5 -----
1 file changed, 5 deletions(-)
diff --git a/common.sh b/common.sh
index 772aee3..829daa7 100644
--- a/common.sh
+++ b/common.sh
@@ -229,7 +229,6 @@ function testShowResults {
function testCreateJUnit() {
mkdir "_junit"
local XML="_junit/result.xml"
- set -x
echo "<?xml version='1.0' encoding='UTF-8'?>" > $XML
echo "<testsuites>" >> $XML
@@ -273,8 +272,4 @@ function testCreateJUnit() {
done
echo '</testsuites>' >> $XML
- set +x
- echo
- cat $XML
- echo
}
--
GitLab