CI: Add script for synchronization of the local mirrors for submodules

    You should use it manually when any submodule in the esp-idf has changed

    You can mark to exclude some of them in mirror-list.txt

diff --git a/tools/ci/mirror-list.txt b/tools/ci/mirror-list.txt
index 15576eb10010f40fd883c21fbd88e9565c6698fb..549359f60e175bd2d5365d994f450aa3d14ff9bc 100644 (file)
--- a/tools/ci/mirror-list.txt
+++ b/tools/ci/mirror-list.txt
@@ -1,10 +1,10 @@
 components/esp32/lib                                @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git
 components/bt/lib                                   @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git
-components/aws_iot/aws-iot-device-sdk-embedded-C    @GENERAL_MIRROR_SERVER@/idf/aws-iot-device-sdk-embedded-C.git
-components/coap/libcoap                             @GENERAL_MIRROR_SERVER@/idf/libcoap.git
-components/esptool_py/esptool                       @GENERAL_MIRROR_SERVER@/idf/esptool.git
-components/libsodium/libsodium                      @GENERAL_MIRROR_SERVER@/idf/libsodium.git
-components/micro-ecc/micro-ecc                      @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git
-components/nghttp/nghttp2                           @GENERAL_MIRROR_SERVER@/idf/nghttp2.git
-third-party/mruby                                   @GENERAL_MIRROR_SERVER@/idf/mruby.git
-third-party/neverbleed                              @GENERAL_MIRROR_SERVER@/idf/neverbleed.git
+components/aws_iot/aws-iot-device-sdk-embedded-C    @GENERAL_MIRROR_SERVER@/idf/aws-iot-device-sdk-embedded-C.git   ALLOW_TO_SYNC_FROM_PUBLIC
+components/coap/libcoap                             @GENERAL_MIRROR_SERVER@/idf/libcoap.git                         ALLOW_TO_SYNC_FROM_PUBLIC
+components/esptool_py/esptool                       @GENERAL_MIRROR_SERVER@/idf/esptool.git                         ALLOW_TO_SYNC_FROM_PUBLIC
+components/libsodium/libsodium                      @GENERAL_MIRROR_SERVER@/idf/libsodium.git                       ALLOW_TO_SYNC_FROM_PUBLIC
+components/micro-ecc/micro-ecc                      @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git                       ALLOW_TO_SYNC_FROM_PUBLIC
+components/nghttp/nghttp2                           @GENERAL_MIRROR_SERVER@/idf/nghttp2.git                         ALLOW_TO_SYNC_FROM_PUBLIC
+third-party/mruby                                   @GENERAL_MIRROR_SERVER@/idf/mruby.git                           ALLOW_TO_SYNC_FROM_PUBLIC
+third-party/neverbleed                              @GENERAL_MIRROR_SERVER@/idf/neverbleed.git                      ALLOW_TO_SYNC_FROM_PUBLIC

diff --git a/tools/ci/mirror-synchronize.sh b/tools/ci/mirror-synchronize.sh
new file mode 100755 (executable)
index 0000000..163e7db
--- /dev/null
+++ b/tools/ci/mirror-synchronize.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Script for automate mirroring
+# You can run this script manually
+#
+
+# -----------------------------------------------------------------------------
+# Common bash
+
+if [[ ! -z ${DEBUG} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+set -o nounset # Exit if variable not set.
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+# -----------------------------------------------------------------------------
+
+[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
+
+REPO_DIR=${1:-"${PWD}"}
+REPO_DIR=$(readlink -f -- "${REPO_DIR}")
+
+SCRIPT_DIR=$(dirname -- "${0}")
+SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
+
+SCRIPT_SH=$(readlink -f -- "${0}")
+
+MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
+
+[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
+[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
+[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
+
+pushd ${REPO_DIR} >/dev/null
+
+# 0
+[ -f ".gitmodules" ] || exit 0
+
+# 1
+# Recursion
+git submodule update --init
+git submodule foreach "${SCRIPT_SH}" # No '--recursive'
+
+# 2
+git submodule deinit --force .
+git submodule update --init
+
+
+# 3
+# Mirroring from original URLs to ones listed in the MIRRORLIST
+
+#   SED parses the strings like:
+#
+#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
+#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
+#
+for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
+do
+    SUBURL=$(git config -f .gitmodules --get submodule.$SUBPATH.url)
+    SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
+    [ ${SUBMIRROR} ] || continue
+    SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
+
+    ALLOW_TO_SYNC=$(join -o"2.3" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
+    if [ "${ALLOW_TO_SYNC}" != "ALLOW_TO_SYNC_FROM_PUBLIC" ]
+    then
+        echo "[should not to sync mirror] ${SUBMIRROR}"
+        continue
+    fi
+
+    echo -e "[mirror sync] ${SUBURL} \tto\t ${SUBMIRROR}"
+
+    pushd ${SUBPATH} >/dev/null
+        git fetch --prune
+        git push --prune ${SUBMIRROR} +refs/remotes/origin/*:refs/heads/* +refs/tags/*:refs/tags/*
+    popd >/dev/null
+done
+
+popd >/dev/null