CI: Fix the getting sources

    Fix the getting sources for a case if the required commit is missed
    in the remote submodule repository.

    Also add more diagnostic.

diff --git a/tools/ci/get-full-sources.sh b/tools/ci/get-full-sources.sh
index 8ebaebbf31b11ec077a3399c8562caaece7e13ce..932948c3ea12733bb71e8c9fb22657e86c1f8d8b 100755 (executable)
--- a/tools/ci/get-full-sources.sh
+++ b/tools/ci/get-full-sources.sh
@@ -10,32 +10,54 @@
 # This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone
 #
 
+# -----------------------------------------------------------------------------
+# Common bash
+if [[ ! -z ${DEBUG} ]]; then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+set -o nounset # Exit if variable not set.
+
 die() {
     echo "${1:-"Unknown Error"}" 1>&2
-    exit 1
+    exit ${2:-1}
 }
+# -----------------------------------------------------------------------------
 
 [ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner."
+[ -z ${GITLAB_SSH_SERVER} ] && die "GITLAB_SSH_SERVER should be defined to run mirror-submodule-update.sh"
+[ -z ${CI_REPOSITORY_URL} ] && die "CI_REPOSITORY_URL should be defined to run mirror-submodule-update.sh"
+[ -z ${CI_COMMIT_SHA} ] && die "CI_COMMIT_SHA should be defined to run mirror-submodule-update.sh"
 [[ ( -z ${IS_PRIVATE} ) && ( -z ${IS_PUBLIC} ) ]] && die "IS_PRIVATE or IS_PUBLIC should be defined in the CI environment."
 
+ERR_CANNOT_UPDATE=13
+
 SCRIPT_DIR=$(dirname -- "${0}")
 update_submodules() {
     if [ "${IS_PRIVATE}" ]; then
-        ${SCRIPT_DIR}/mirror-submodule-update.sh
+        ${SCRIPT_DIR}/mirror-submodule-update.sh || return $?
     else
-        git submodule foreach "git submodule deinit --force ."
-        git submodule deinit --force .
         git submodule update --init --recursive
     fi
 }
 
-DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX)
 del_files() {
+    DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX)
     # if non-empty
     [ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" )
+    trap 'del_files_rollback' ERR
 }
 del_files_confirm() {
-    rm -rf "${DELETED_FILES}"
+    [ -d "${DELETED_FILES}" ] && rm -rf "${DELETED_FILES}"
+    trap ERR
+}
+del_files_rollback() {
+    [ "$(ls -A .)" ] && [ "$(ls -A ${DELETED_FILES}/)" ] && ( shopt -s dotglob; rm -rf * )
+    [ "$(ls -A ${DELETED_FILES}/)" ] && ( shopt -s dotglob; mv "${DELETED_FILES}/"* . )
+    [ -d "${DELETED_FILES}" ] && rmdir "${DELETED_FILES}"
+    trap ERR
 }
 
 RETRIES=10
@@ -47,6 +69,9 @@ for try in `seq $RETRIES`; do
     update_submodules &&
         echo "Fetch strategy submodules succeeded" &&
         exit 0
+
+    git submodule foreach "git reset --hard HEAD && git submodule deinit --force ."
+    git submodule deinit --force .
 done
 
 # Then we use the clean way.
@@ -60,8 +85,19 @@ for try in `seq $RETRIES`; do
         echo "Clone strategy succeeded" &&
         del_files_confirm &&
         exit 0
-
+    ERR_RES=$?
+    del_files_rollback
     echo "Clean clone failed..."
+    if [ $ERR_RES -eq $ERR_CANNOT_UPDATE ]; then
+        echo "###"
+        echo "### If you have updated one of the submodules,"
+        echo "### you have to synchronize the local mirrors manually"
+        echo "###"
+        echo "###     https://gitlab.espressif.cn:6688/idf/esp-idf/wikis/ci-use-guide#submodule-mirroring-for-private-branches"
+        echo "###"
+
+        die "Failed to clone repo & submodules together" $ERR_RES
+    fi
 done
 
 die "Failed to clone repo & submodules together"

diff --git a/tools/ci/mirror-submodule-update.sh b/tools/ci/mirror-submodule-update.sh
index 38fa43dec2d7658d8384ec1818ed9e5a1c3b6bed..04471f143e3a7a90e85a3fcaabb62fac5c274796 100755 (executable)
--- a/tools/ci/mirror-submodule-update.sh
+++ b/tools/ci/mirror-submodule-update.sh
@@ -26,6 +26,8 @@ die() {
 
 [ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
 
+ERR_CANNOT_UPDATE=13
+
 REPO_DIR=${1:-"${PWD}"}
 REPO_DIR=$(readlink -f -- "${REPO_DIR}")
 
@@ -70,7 +72,7 @@ done
 
 # 3
 # Getting submodules of the current repository from the local mirrors
-git submodule update
+git submodule update || exit $ERR_CANNOT_UPDATE
 
 # 4
 # Replacing URLs for each sub-submodule.