]> granicus.if.org Git - esp-idf/commitdiff
tools: add metadata file and idf_tools.py
authorIvan Grokhotkov <ivan@espressif.com>
Wed, 17 Apr 2019 07:30:30 +0000 (15:30 +0800)
committerIvan Grokhotkov <ivan@espressif.com>
Fri, 24 May 2019 09:04:23 +0000 (17:04 +0800)
.gitlab-ci.yml
tools/ci/executable-list.txt
tools/idf_tools.py [new file with mode: 0755]
tools/test_idf_tools/test_idf_tools.py [new file with mode: 0755]
tools/tools.json [new file with mode: 0644]
tools/tools_schema.json [new file with mode: 0644]

index 48a9624ef39e8e0da51b54eecd02240a0cf2bbfc..e855c86feb52f05b814c1c732c0c9fd2b7afec2b 100644 (file)
@@ -601,6 +601,14 @@ test_idf_size:
     - cd ${IDF_PATH}/tools/test_idf_size
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
 
+test_idf_tools:
+  <<: *host_test_template
+  script:
+    # Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment
+    - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/opt/espressif${CUSTOM_TOOLCHAIN_PATH:+\|${CUSTOM_TOOLCHAIN_PATH}}" | tr "\n" ":"); echo ${p%:})
+    - cd ${IDF_PATH}/tools/test_idf_tools
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
+
 test_esp_err_to_name_on_host:
   <<: *host_test_template
   artifacts:
index 3324c758a781ad407600ad819fc029863bb4b126..5cbf7b3a2b5c13f6579b895c0fefe4914b243aeb 100644 (file)
@@ -53,6 +53,7 @@ tools/gen_esp_err_to_name.py
 tools/idf.py
 tools/idf_monitor.py
 tools/idf_size.py
+tools/idf_tools.py
 tools/kconfig/check.sh
 tools/kconfig/lxdialog/check-lxdialog.sh
 tools/kconfig/merge_config.sh
@@ -67,6 +68,7 @@ tools/mass_mfg/mfg_gen.py
 tools/test_check_kconfigs.py
 tools/test_idf_monitor/run_test_idf_monitor.py
 tools/test_idf_size/test.sh
+tools/test_idf_tools/test_idf_tools.py
 tools/unit-test-app/unit_test.py
 tools/windows/eclipse_make.sh
 tools/windows/tool_setup/build_installer.sh
diff --git a/tools/idf_tools.py b/tools/idf_tools.py
new file mode 100755 (executable)
index 0000000..362a5f4
--- /dev/null
@@ -0,0 +1,1243 @@
+#!/usr/bin/env python
+# coding=utf-8
+#
+# This script helps installing tools required to use the ESP-IDF, and updating PATH
+# to use the installed tools. It can also create a Python virtual environment,
+# and install Python requirements into it.
+#  It does not install OS dependencies. It does install tools such as the Xtensa
+#  GCC toolchain and ESP32 ULP coprocessor toolchain.
+#
+# By default, downloaded tools will be installed under $HOME/.espressif directory
+# (%USERPROFILE%/.espressif on Windows). This path can be modified by setting
+# IDF_TOOLS_PATH variable prior to running this tool.
+#
+# Users do not need to interact with this script directly. In IDF root directory,
+# install.sh (.bat) and export.sh (.bat) scripts are provided to invoke this script.
+#
+# Usage:
+#
+# * To install the tools, run `idf_tools.py install`.
+#
+# * To install the Python environment, run `idf_tools.py install-python-env`.
+#
+# * To start using the tools, run `eval "$(idf_tools.py export)"` — this will update
+#   the PATH to point to the installed tools and set up other environment variables
+#   needed by the tools.
+#
+###
+#
+# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import subprocess
+import sys
+import argparse
+import re
+import platform
+import hashlib
+import tarfile
+import zipfile
+import errno
+import shutil
+import functools
+from collections import OrderedDict, namedtuple
+
+try:
+    from urllib.request import urlretrieve
+except ImportError:
+    from urllib import urlretrieve
+
+
+TOOLS_FILE = 'tools/tools.json'
+TOOLS_SCHEMA_FILE = 'tools/tools_schema.json'
+TOOLS_FILE_NEW = 'tools/tools.new.json'
+TOOLS_FILE_VERSION = 1
+IDF_TOOLS_PATH_DEFAULT = os.path.join('~', '.espressif')
+UNKNOWN_VERSION = 'unknown'
+SUBST_TOOL_PATH_REGEX = re.compile(r'\${TOOL_PATH}')
+VERSION_REGEX_REPLACE_DEFAULT = r'\1'
+IDF_MAINTAINER = os.environ.get('IDF_MAINTAINER') or False
+TODO_MESSAGE = 'TODO'
+DOWNLOAD_RETRY_COUNT = 3
+URL_PREFIX_MAP_SEPARATOR = ','
+IDF_TOOLS_INSTALL_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
+IDF_TOOLS_EXPORT_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
+
+PYTHON_PLATFORM = platform.system() + '-' + platform.machine()
+
+# Identifiers used in tools.json for different platforms.
+PLATFORM_WIN32 = 'win32'
+PLATFORM_WIN64 = 'win64'
+PLATFORM_MACOS = 'macos'
+PLATFORM_LINUX32 = 'linux-i686'
+PLATFORM_LINUX64 = 'linux-amd64'
+PLATFORM_LINUX_ARM32 = 'linux-armel'
+PLATFORM_LINUX_ARM64 = 'linux-arm64'
+
+
+# Mappings from various other names these platforms are known as, to the identifiers above.
+# This includes strings produced from "platform.system() + '-' + platform.machine()", see PYTHON_PLATFORM
+# definition above.
+# This list also includes various strings used in release archives of xtensa-esp32-elf-gcc, OpenOCD, etc.
+PLATFORM_FROM_NAME = {
+    # Windows
+    PLATFORM_WIN32: PLATFORM_WIN32,
+    'Windows-i686': PLATFORM_WIN32,
+    PLATFORM_WIN64: PLATFORM_WIN64,
+    'Windows-x86_64': PLATFORM_WIN64,
+    'Windows-AMD64': PLATFORM_WIN64,
+    # macOS
+    PLATFORM_MACOS: PLATFORM_MACOS,
+    'osx': PLATFORM_MACOS,
+    'darwin': PLATFORM_MACOS,
+    'Darwin-x86_64': PLATFORM_MACOS,
+    # Linux
+    PLATFORM_LINUX64: PLATFORM_LINUX64,
+    'linux64': PLATFORM_LINUX64,
+    'Linux-x86_64': PLATFORM_LINUX64,
+    PLATFORM_LINUX32: PLATFORM_LINUX32,
+    'linux32': PLATFORM_LINUX32,
+    'Linux-i686': PLATFORM_LINUX32,
+    PLATFORM_LINUX_ARM32: PLATFORM_LINUX_ARM32,
+    'Linux-arm': PLATFORM_LINUX_ARM32,
+    'Linux-armv7l': PLATFORM_LINUX_ARM32,
+    PLATFORM_LINUX_ARM64: PLATFORM_LINUX_ARM64,
+    'Linux-arm64': PLATFORM_LINUX_ARM64,
+    'Linux-aarch64': PLATFORM_LINUX_ARM64,
+    'Linux-armv8l': PLATFORM_LINUX_ARM64,
+}
+
+UNKNOWN_PLATFORM = 'unknown'
+CURRENT_PLATFORM = PLATFORM_FROM_NAME.get(PYTHON_PLATFORM, UNKNOWN_PLATFORM)
+
+EXPORT_SHELL = 'shell'
+EXPORT_KEY_VALUE = 'key-value'
+
+
+global_quiet = False
+global_idf_path = None
+global_idf_tools_path = None
+global_tools_json = None
+
+
+def fatal(text, *args):
+    if not global_quiet:
+        sys.stderr.write('ERROR: ' + text + '\n', *args)
+
+
+def warn(text, *args):
+    if not global_quiet:
+        sys.stderr.write('WARNING: ' + text + '\n', *args)
+
+
+def info(text, f=None, *args):
+    if not global_quiet:
+        if f is None:
+            f = sys.stdout
+        f.write(text + '\n', *args)
+
+
+def run_cmd_check_output(cmd, input_text=None, extra_paths=None):
+    # If extra_paths is given, locate the executable in one of these directories.
+    # Note: it would seem logical to add extra_paths to env[PATH], instead, and let OS do the job of finding the
+    # executable for us. However this does not work on Windows: https://bugs.python.org/issue8557.
+    if extra_paths:
+        found = False
+        extensions = ['']
+        if sys.platform == 'win32':
+            extensions.append('.exe')
+        for path in extra_paths:
+            for ext in extensions:
+                fullpath = os.path.join(path, cmd[0] + ext)
+                if os.path.exists(fullpath):
+                    cmd[0] = fullpath
+                    found = True
+                    break
+            if found:
+                break
+
+    try:
+        if input_text:
+            input_text = input_text.encode()
+        result = subprocess.run(cmd, capture_output=True, check=True, input=input_text)
+        return result.stdout + result.stderr
+    except AttributeError:
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate(input_text)
+        if p.returncode != 0:
+            try:
+                raise subprocess.CalledProcessError(p.returncode, cmd, stdout, stderr)
+            except TypeError:
+                raise subprocess.CalledProcessError(p.returncode, cmd, stdout)
+        return stdout + stderr
+
+
+def to_shell_specific_paths(paths_list):
+    if sys.platform == 'win32':
+        paths_list = [p.replace('/', os.path.sep) if os.path.sep in p else p for p in paths_list]
+
+        if 'MSYSTEM' in os.environ:
+            paths_msys = run_cmd_check_output(['cygpath', '-u', '-f', '-'],
+                                              input_text='\n'.join(paths_list))
+            paths_list = paths_msys.decode().strip().split('\n')
+
+    return paths_list
+
+
+def get_env_for_extra_paths(extra_paths):
+    """
+    Return a copy of environment variables dict, prepending paths listed in extra_paths
+    to the PATH environment variable.
+    """
+    env_arg = os.environ.copy()
+    new_path = os.pathsep.join(extra_paths) + os.pathsep + env_arg['PATH']
+    if sys.version_info.major == 2:
+        env_arg['PATH'] = new_path.encode('utf8')
+    else:
+        env_arg['PATH'] = new_path
+    return env_arg
+
+
+def get_file_size_sha256(filename, block_size=65536):
+    sha256 = hashlib.sha256()
+    size = 0
+    with open(filename, 'rb') as f:
+        for block in iter(lambda: f.read(block_size), b''):
+            sha256.update(block)
+            size += len(block)
+    return size, sha256.hexdigest()
+
+
+def report_progress(count, block_size, total_size):
+    percent = int(count * block_size * 100 / total_size)
+    percent = min(100, percent)
+    sys.stdout.write("\r%d%%" % percent)
+    sys.stdout.flush()
+
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno != errno.EEXIST or not os.path.isdir(path):
+            raise
+
+
+def unpack(filename, destination):
+    info('Extracting {0} to {1}'.format(filename, destination))
+    if filename.endswith('tar.gz'):
+        archive_obj = tarfile.open(filename, 'r:gz')
+    elif filename.endswith('zip'):
+        archive_obj = zipfile.ZipFile(filename)
+    else:
+        raise NotImplementedError('Unsupported archive type')
+    if sys.version_info.major == 2:
+        # This is a workaround for the issue that unicode destination is not handled:
+        # https://bugs.python.org/issue17153
+        destination = str(destination)
+    archive_obj.extractall(destination)
+
+
+def strip_container_dirs(path, levels):
+    assert levels > 0
+    # move the original directory out of the way (add a .tmp suffix)
+    tmp_path = path + '.tmp'
+    if os.path.exists(tmp_path):
+        shutil.rmtree(tmp_path)
+    os.rename(path, tmp_path)
+    os.mkdir(path)
+    base_path = tmp_path
+    # walk given number of levels down
+    for level in range(levels):
+        contents = os.listdir(base_path)
+        if len(contents) > 1:
+            raise RuntimeError('at level {}, expected 1 entry, got {}'.format(level, contents))
+        base_path = os.path.join(base_path, contents[0])
+        if not os.path.isdir(base_path):
+            raise RuntimeError('at level {}, {} is not a directory'.format(level, contents[0]))
+    # get the list of directories/files to move
+    contents = os.listdir(base_path)
+    for name in contents:
+        move_from = os.path.join(base_path, name)
+        move_to = os.path.join(path, name)
+        os.rename(move_from, move_to)
+    shutil.rmtree(tmp_path)
+
+
+class ToolNotFound(RuntimeError):
+    pass
+
+
+class ToolExecError(RuntimeError):
+    pass
+
+
+class DownloadError(RuntimeError):
+    pass
+
+
+class IDFToolDownload(object):
+    def __init__(self, platform_name, url, size, sha256):
+        self.platform_name = platform_name
+        self.url = url
+        self.size = size
+        self.sha256 = sha256
+        self.platform_name = platform_name
+
+
+@functools.total_ordering
+class IDFToolVersion(object):
+    STATUS_RECOMMENDED = 'recommended'
+    STATUS_SUPPORTED = 'supported'
+    STATUS_DEPRECATED = 'deprecated'
+
+    STATUS_VALUES = [STATUS_RECOMMENDED, STATUS_SUPPORTED, STATUS_DEPRECATED]
+
+    def __init__(self, version, status):
+        self.version = version
+        self.status = status
+        self.downloads = OrderedDict()
+        self.latest = False
+
+    def __lt__(self, other):
+        if self.status != other.status:
+            return self.status > other.status
+        else:
+            assert not (self.status == IDFToolVersion.STATUS_RECOMMENDED
+                        and other.status == IDFToolVersion.STATUS_RECOMMENDED)
+            return self.version < other.version
+
+    def __eq__(self, other):
+        return self.status == other.status and self.version == other.version
+
+    def add_download(self, platform_name, url, size, sha256):
+        self.downloads[platform_name] = IDFToolDownload(platform_name, url, size, sha256)
+
+    def get_download_for_platform(self, platform_name):
+        if platform_name in PLATFORM_FROM_NAME.keys():
+            platform_name = PLATFORM_FROM_NAME[platform_name]
+        if platform_name in self.downloads.keys():
+            return self.downloads[platform_name]
+        if 'any' in self.downloads.keys():
+            return self.downloads['any']
+        return None
+
+    def compatible_with_platform(self, platform_name=PYTHON_PLATFORM):
+        return self.get_download_for_platform(platform_name) is not None
+
+
+OPTIONS_LIST = ['version_cmd',
+                'version_regex',
+                'version_regex_replace',
+                'export_paths',
+                'export_vars',
+                'install',
+                'info_url',
+                'license',
+                'strip_container_dirs']
+
+IDFToolOptions = namedtuple('IDFToolOptions', OPTIONS_LIST)
+
+
+class IDFTool(object):
+    # possible values of 'install' field
+    INSTALL_ALWAYS = 'always'
+    INSTALL_ON_REQUEST = 'on_request'
+    INSTALL_NEVER = 'never'
+
+    def __init__(self, name, description, install, info_url, license, version_cmd, version_regex, version_regex_replace=None,
+                 strip_container_dirs=0):
+        self.name = name
+        self.description = description
+        self.versions = OrderedDict()
+        self.version_in_path = None
+        self.versions_installed = []
+        if version_regex_replace is None:
+            version_regex_replace = VERSION_REGEX_REPLACE_DEFAULT
+        self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace,
+                                      [], OrderedDict(), install, info_url, license, strip_container_dirs)
+        self.platform_overrides = []
+        self._update_current_options()
+
+    def _update_current_options(self):
+        self._current_options = IDFToolOptions(*self.options)
+        for override in self.platform_overrides:
+            if CURRENT_PLATFORM not in override['platforms']:
+                continue
+            override_dict = override.copy()
+            del override_dict['platforms']
+            self._current_options = self._current_options._replace(**override_dict)
+
+    def add_version(self, version):
+        assert(type(version) is IDFToolVersion)
+        self.versions[version.version] = version
+
+    def get_path(self):
+        return os.path.join(global_idf_tools_path, 'tools', self.name)
+
+    def get_path_for_version(self, version):
+        assert(version in self.versions)
+        return os.path.join(self.get_path(), version)
+
+    def get_export_paths(self, version):
+        tool_path = self.get_path_for_version(version)
+        return [os.path.join(tool_path, *p) for p in self._current_options.export_paths]
+
+    def get_export_vars(self, version):
+        """
+        Get the dictionary of environment variables to be exported, for the given version.
+        Expands:
+        - ${TOOL_PATH} => the actual path where the version is installed
+        """
+        result = {}
+        for k, v in self._current_options.export_vars.items():
+            replace_path = self.get_path_for_version(version).replace('\\', '\\\\')
+            v_repl = re.sub(SUBST_TOOL_PATH_REGEX, replace_path, v)
+            if v_repl != v:
+                v_repl = to_shell_specific_paths([v_repl])[0]
+            result[k] = v_repl
+        return result
+
+    def check_version(self, extra_paths=None):
+        """
+        Execute the tool, optionally prepending extra_paths to PATH,
+        extract the version string and return it as a result.
+        Raises ToolNotFound if the tool is not found (not present in the paths).
+        Raises ToolExecError if the tool returns with a non-zero exit code.
+        Returns 'unknown' if tool returns something from which version string
+        can not be extracted.
+        """
+        cmd = self._current_options.version_cmd
+        try:
+            version_cmd_result = run_cmd_check_output(cmd, None, extra_paths)
+        except OSError:
+            # tool is not on the path
+            raise ToolNotFound('Tool {} not found'.format(self.name))
+        except subprocess.CalledProcessError as e:
+            raise ToolExecError('Command {} has returned non-zero exit code ({})\n'.format(
+                ' '.join(self._current_options.version_cmd), e.returncode))
+
+        in_str = version_cmd_result.decode()
+        match = re.search(self._current_options.version_regex, in_str)
+        if not match:
+            return UNKNOWN_VERSION
+        return re.sub(self._current_options.version_regex, self._current_options.version_regex_replace, match.group(0))
+
+    def get_install_type(self):
+        return self._current_options.install
+
+    def get_recommended_version(self):
+        recommended_versions = [k for k, v in self.versions.items()
+                                if v.status == IDFToolVersion.STATUS_RECOMMENDED
+                                and v.compatible_with_platform()]
+        assert len(recommended_versions) <= 1
+        if recommended_versions:
+            return recommended_versions[0]
+        return None
+
+    def get_preferred_installed_version(self):
+        recommended_versions = [k for k in self.versions_installed
+                                if self.versions[k].status == IDFToolVersion.STATUS_RECOMMENDED
+                                and self.versions[k].compatible_with_platform()]
+        assert len(recommended_versions) <= 1
+        if recommended_versions:
+            return recommended_versions[0]
+        return None
+
+    def find_installed_versions(self):
+        """
+        Checks whether the tool can be found in PATH and in global_idf_tools_path.
+        Writes results to self.version_in_path and self.versions_installed.
+        """
+        # First check if the tool is in system PATH
+        try:
+            ver_str = self.check_version()
+        except ToolNotFound:
+            # not in PATH
+            pass
+        except ToolExecError:
+            warn('tool {} found in path, but failed to run'.format(self.name))
+        else:
+            self.version_in_path = ver_str
+
+        # Now check all the versions installed in global_idf_tools_path
+        self.versions_installed = []
+        for version, version_obj in self.versions.items():
+            if not version_obj.compatible_with_platform():
+                continue
+            tool_path = self.get_path_for_version(version)
+            if not os.path.exists(tool_path):
+                # version not installed
+                continue
+            try:
+                ver_str = self.check_version(self.get_export_paths(version))
+            except ToolNotFound:
+                warn('directory for tool {} version {} is present, but tool was not found'.format(
+                    self.name, version))
+            except ToolExecError:
+                warn('tool {} version {} is installed, but the tool failed to run'.format(
+                    self.name, version))
+            else:
+                if ver_str != version:
+                    warn('tool {} version {} is installed, but has reported version {}'.format(
+                        self.name, version, ver_str))
+                else:
+                    self.versions_installed.append(version)
+
+    def download(self, version):
+        assert(version in self.versions)
+        download_obj = self.versions[version].get_download_for_platform(PYTHON_PLATFORM)
+        if not download_obj:
+            fatal('No packages for tool {} platform {}!'.format(self.name, PYTHON_PLATFORM))
+            raise DownloadError()
+
+        url = download_obj.url
+        archive_name = os.path.basename(url)
+        local_path = os.path.join(global_idf_tools_path, 'dist', archive_name)
+        mkdir_p(os.path.dirname(local_path))
+
+        if os.path.isfile(local_path):
+            if not self.check_download_file(download_obj, local_path):
+                warn('removing downloaded file {0} and downloading again'.format(archive_name))
+                os.unlink(local_path)
+            else:
+                info('file {0} is already downloaded'.format(archive_name))
+                return
+
+        downloaded = False
+        for retry in range(DOWNLOAD_RETRY_COUNT):
+            local_temp_path = local_path + '.tmp'
+            info('Downloading {} to {}'.format(archive_name, local_temp_path))
+            urlretrieve(url, local_temp_path, report_progress)
+            sys.stdout.write("\rDone\n")
+            sys.stdout.flush()
+            if not self.check_download_file(download_obj, local_temp_path):
+                warn('Failed to download file {}'.format(local_temp_path))
+                continue
+            os.rename(local_temp_path, local_path)
+            downloaded = True
+            break
+        if not downloaded:
+            fatal('Failed to download, and retry count has expired')
+            raise DownloadError()
+
+    def install(self, version):
+        # Currently this is called after calling 'download' method, so here are a few asserts
+        # for the conditions which should be true once that method is done.
+        assert (version in self.versions)
+        download_obj = self.versions[version].get_download_for_platform(PYTHON_PLATFORM)
+        assert (download_obj is not None)
+        archive_name = os.path.basename(download_obj.url)
+        archive_path = os.path.join(global_idf_tools_path, 'dist', archive_name)
+        assert (os.path.isfile(archive_path))
+        dest_dir = self.get_path_for_version(version)
+        if os.path.exists(dest_dir):
+            warn('destination path already exists, removing')
+            shutil.rmtree(dest_dir)
+        mkdir_p(dest_dir)
+        unpack(archive_path, dest_dir)
+        if self._current_options.strip_container_dirs:
+            strip_container_dirs(dest_dir, self._current_options.strip_container_dirs)
+
+    @staticmethod
+    def check_download_file(download_obj, local_path):
+        expected_sha256 = download_obj.sha256
+        expected_size = download_obj.size
+        file_size, file_sha256 = get_file_size_sha256(local_path)
+        if file_size != expected_size:
+            warn('file size mismatch for {0}'.format(local_path))
+            return False
+        if file_sha256 != expected_sha256:
+            warn('hash mismatch for {0}'.format(local_path))
+            return False
+        return True
+
+    @classmethod
+    def from_json(cls, tool_dict):
+        # json.load will return 'str' types in Python 3 and 'unicode' in Python 2
+        expected_str_type = type(u'')
+
+        # Validate json fields
+        tool_name = tool_dict.get('name')
+        if type(tool_name) is not expected_str_type:
+            raise RuntimeError('tool_name is not a string')
+
+        description = tool_dict.get('description')
+        if type(description) is not expected_str_type:
+            raise RuntimeError('description is not a string')
+
+        version_cmd = tool_dict.get('version_cmd')
+        if type(version_cmd) is not list:
+            raise RuntimeError('version_cmd for tool %s is not a list of strings' % tool_name)
+
+        version_regex = tool_dict.get('version_regex')
+        if type(version_regex) is not expected_str_type or not version_regex:
+            raise RuntimeError('version_regex for tool %s is not a non-empty string' % tool_name)
+
+        version_regex_replace = tool_dict.get('version_regex_replace')
+        if version_regex_replace and type(version_regex_replace) is not expected_str_type:
+            raise RuntimeError('version_regex_replace for tool %s is not a string' % tool_name)
+
+        export_paths = tool_dict.get('export_paths')
+        if type(export_paths) is not list:
+            raise RuntimeError('export_paths for tool %s is not a list' % tool_name)
+
+        export_vars = tool_dict.get('export_vars', {})
+        if type(export_vars) is not dict:
+            raise RuntimeError('export_vars for tool %s is not a mapping' % tool_name)
+
+        versions = tool_dict.get('versions')
+        if type(versions) is not list:
+            raise RuntimeError('versions for tool %s is not an array' % tool_name)
+
+        install = tool_dict.get('install', False)
+        if type(install) is not expected_str_type:
+            raise RuntimeError('install for tool %s is not a string' % tool_name)
+
+        info_url = tool_dict.get('info_url', False)
+        if type(info_url) is not expected_str_type:
+            raise RuntimeError('info_url for tool %s is not a string' % tool_name)
+
+        license = tool_dict.get('license', False)
+        if type(license) is not expected_str_type:
+            raise RuntimeError('license for tool %s is not a string' % tool_name)
+
+        strip_container_dirs = tool_dict.get('strip_container_dirs', 0)
+        if strip_container_dirs and type(strip_container_dirs) is not int:
+            raise RuntimeError('strip_container_dirs for tool %s is not an int' % tool_name)
+
+        overrides_list = tool_dict.get('platform_overrides', [])
+        if type(overrides_list) is not list:
+            raise RuntimeError('platform_overrides for tool %s is not a list' % tool_name)
+
+        # Create the object
+        tool_obj = cls(tool_name, description, install, info_url, license,
+                       version_cmd, version_regex, version_regex_replace,
+                       strip_container_dirs)
+
+        for path in export_paths:
+            tool_obj.options.export_paths.append(path)
+
+        for name, value in export_vars.items():
+            tool_obj.options.export_vars[name] = value
+
+        for index, override in enumerate(overrides_list):
+            platforms_list = override.get('platforms')
+            if type(platforms_list) is not list:
+                raise RuntimeError('platforms for override %d of tool %s is not a list' % (index, tool_name))
+
+            install = override.get('install')
+            if install is not None and type(install) is not expected_str_type:
+                raise RuntimeError('install for override %d of tool %s is not a string' % (index, tool_name))
+
+            version_cmd = override.get('version_cmd')
+            if version_cmd is not None and type(version_cmd) is not list:
+                raise RuntimeError('version_cmd for override %d of tool %s is not a list of strings' %
+                                   (index, tool_name))
+
+            version_regex = override.get('version_regex')
+            if version_regex is not None and (type(version_regex) is not expected_str_type or not version_regex):
+                raise RuntimeError('version_regex for override %d of tool %s is not a non-empty string' %
+                                   (index, tool_name))
+
+            version_regex_replace = override.get('version_regex_replace')
+            if version_regex_replace is not None and type(version_regex_replace) is not expected_str_type:
+                raise RuntimeError('version_regex_replace for override %d of tool %s is not a string' %
+                                   (index, tool_name))
+
+            export_paths = override.get('export_paths')
+            if export_paths is not None and type(export_paths) is not list:
+                raise RuntimeError('export_paths for override %d of tool %s is not a list' % (index, tool_name))
+
+            export_vars = override.get('export_vars')
+            if export_vars is not None and type(export_vars) is not dict:
+                raise RuntimeError('export_vars for override %d of tool %s is not a mapping' % (index, tool_name))
+            tool_obj.platform_overrides.append(override)
+
+        recommended_versions = {}
+        for version_dict in versions:
+            version = version_dict.get('name')
+            if type(version) is not expected_str_type:
+                raise RuntimeError('version name for tool {} is not a string'.format(tool_name))
+
+            version_status = version_dict.get('status')
+            if type(version_status) is not expected_str_type and version_status not in IDFToolVersion.STATUS_VALUES:
+                raise RuntimeError('tool {} version {} status is not one of {}', tool_name, version,
+                                   IDFToolVersion.STATUS_VALUES)
+
+            version_obj = IDFToolVersion(version, version_status)
+            for platform_id, platform_dict in version_dict.items():
+                if platform_id in ['name', 'status']:
+                    continue
+                if platform_id not in PLATFORM_FROM_NAME.keys():
+                    raise RuntimeError('invalid platform %s for tool %s version %s' %
+                                       (platform_id, tool_name, version))
+
+                version_obj.add_download(platform_id,
+                                         platform_dict['url'], platform_dict['size'], platform_dict['sha256'])
+
+                if version_status == IDFToolVersion.STATUS_RECOMMENDED:
+                    if platform_id not in recommended_versions:
+                        recommended_versions[platform_id] = []
+                    recommended_versions[platform_id].append(version)
+
+            tool_obj.add_version(version_obj)
+        for platform_id, version_list in recommended_versions.items():
+            if len(version_list) > 1:
+                raise RuntimeError('tool {} for platform {} has {} recommended versions'.format(
+                    tool_name, platform_id, len(recommended_versions)))
+            if install != IDFTool.INSTALL_NEVER and len(recommended_versions) == 0:
+                raise RuntimeError('required/optional tool {} for platform {} has no recommended versions'.format(
+                    tool_name, platform_id))
+
+        tool_obj._update_current_options()
+        return tool_obj
+
+    def to_json(self):
+        versions_array = []
+        for version, version_obj in self.versions.items():
+            version_json = {
+                'name': version,
+                'status': version_obj.status
+            }
+            for platform_id, download in version_obj.downloads.items():
+                version_json[platform_id] = {
+                    'url': download.url,
+                    'size': download.size,
+                    'sha256': download.sha256
+                }
+            versions_array.append(version_json)
+        overrides_array = self.platform_overrides
+
+        tool_json = {
+            'name': self.name,
+            'description': self.description,
+            'export_paths': self.options.export_paths,
+            'export_vars': self.options.export_vars,
+            'install': self.options.install,
+            'info_url': self.options.info_url,
+            'license': self.options.license,
+            'version_cmd': self.options.version_cmd,
+            'version_regex': self.options.version_regex,
+            'versions': versions_array,
+        }
+        if self.options.version_regex_replace != VERSION_REGEX_REPLACE_DEFAULT:
+            tool_json['version_regex_replace'] = self.options.version_regex_replace
+        if overrides_array:
+            tool_json['platform_overrides'] = overrides_array
+        if self.options.strip_container_dirs:
+            tool_json['strip_container_dirs'] = self.options.strip_container_dirs
+        return tool_json
+
+
+def load_tools_info():
+    """
+    Load tools metadata from tools.json, return a dictionary: tool name - tool info
+    """
+    tool_versions_file_name = global_tools_json
+
+    with open(tool_versions_file_name, 'r') as f:
+        tools_info = json.load(f)
+
+    return parse_tools_info_json(tools_info)
+
+
+def parse_tools_info_json(tools_info):
+    """
+    Parse and validate the dictionary obtained by loading the tools.json file.
+    Returns a dictionary of tools (key: tool name, value: IDFTool object).
+    """
+    if tools_info['version'] != TOOLS_FILE_VERSION:
+        raise RuntimeError('Invalid version')
+
+    tools_dict = OrderedDict()
+
+    tools_array = tools_info.get('tools')
+    if type(tools_array) is not list:
+        raise RuntimeError('tools property is missing or not an array')
+
+    for tool_dict in tools_array:
+        tool = IDFTool.from_json(tool_dict)
+        tools_dict[tool.name] = tool
+
+    return tools_dict
+
+
+def dump_tools_json(tools_info):
+    tools_array = []
+    for tool_name, tool_obj in tools_info.items():
+        tool_json = tool_obj.to_json()
+        tools_array.append(tool_json)
+    file_json = {'version': TOOLS_FILE_VERSION, 'tools': tools_array}
+    return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True)
+
+
+def get_python_env_path():
+    python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
+
+    version_file_path = os.path.join(global_idf_path, 'version.txt')
+    if os.path.exists(version_file_path):
+        with open(version_file_path, "r") as version_file:
+            idf_version_str = version_file.read()
+    else:
+        idf_version_str = subprocess.check_output(['git', '-C', global_idf_path, 'describe', '--tags'], cwd=global_idf_path, env=os.environ).decode()
+    match = re.match(r'^v([0-9]+\.[0-9]+).*', idf_version_str)
+    idf_version = match.group(1)
+
+    idf_python_env_path = os.path.join(global_idf_tools_path, 'python_env',
+                                       'idf{}_py{}_env'.format(idf_version, python_ver_major_minor))
+
+    if sys.platform == 'win32':
+        subdir = 'Scripts'
+        python_exe = 'python.exe'
+    else:
+        subdir = 'bin'
+        python_exe = 'python'
+
+    idf_python_export_path = os.path.join(idf_python_env_path, subdir)
+    virtualenv_python = os.path.join(idf_python_export_path, python_exe)
+
+    return idf_python_env_path, idf_python_export_path, virtualenv_python
+
+
+def action_list(args):
+    tools_info = load_tools_info()
+    for name, tool in tools_info.items():
+        if tool.get_install_type() == IDFTool.INSTALL_NEVER:
+            continue
+        optional_str = ' (optional)' if tool.get_install_type() == IDFTool.INSTALL_ON_REQUEST else ''
+        info('* {}: {}{}'.format(name, tool.description, optional_str))
+        tool.find_installed_versions()
+        versions_for_platform = {k: v for k, v in tool.versions.items() if v.compatible_with_platform()}
+        if not versions_for_platform:
+            info('  (no versions compatible with platform {})'.format(PYTHON_PLATFORM))
+            continue
+        versions_sorted = sorted(versions_for_platform.keys(), key=tool.versions.get, reverse=True)
+        for version in versions_sorted:
+            version_obj = tool.versions[version]
+            info('  - {} ({}{})'.format(version, version_obj.status,
+                                        ', installed' if version in tool.versions_installed else ''))
+
+
+def action_check(args):
+    tools_info = load_tools_info()
+    not_found_list = []
+    info('Checking for installed tools...')
+    for name, tool in tools_info.items():
+        if tool.get_install_type() == IDFTool.INSTALL_NEVER:
+            continue
+        tool_found_somewhere = False
+        info('Checking tool %s' % name)
+        tool.find_installed_versions()
+        if tool.version_in_path:
+            info('    version found in PATH: %s' % tool.version_in_path)
+            tool_found_somewhere = True
+        else:
+            info('    no version found in PATH')
+
+        for version in tool.versions_installed:
+            info('    version installed in tools directory: %s' % version)
+            tool_found_somewhere = True
+        if not tool_found_somewhere and tool.get_install_type() == IDFTool.INSTALL_ALWAYS:
+            not_found_list.append(name)
+    if not_found_list:
+        fatal('The following required tools were not found: ' + ' '.join(not_found_list))
+        raise SystemExit(1)
+
+
+def action_export(args):
+    tools_info = load_tools_info()
+    all_tools_found = True
+    export_vars = {}
+    paths_to_export = []
+    for name, tool in tools_info.items():
+        if tool.get_install_type() == IDFTool.INSTALL_NEVER:
+            continue
+        tool.find_installed_versions()
+
+        if tool.version_in_path:
+            if tool.version_in_path not in tool.versions:
+                # unsupported version
+                if args.prefer_system:
+                    warn('using an unsupported version of tool {} found in PATH: {}'.format(
+                        tool.name, tool.version_in_path))
+                    continue
+                else:
+                    # unsupported version in path
+                    pass
+            else:
+                # supported/deprecated version in PATH, use it
+                version_obj = tool.versions[tool.version_in_path]
+                if version_obj.status == IDFToolVersion.STATUS_SUPPORTED:
+                    info('Using a supported version of tool {} found in PATH: {}.'.format(name, tool.version_in_path),
+                         f=sys.stderr)
+                    info('However the recommended version is {}.'.format(tool.get_recommended_version()),
+                         f=sys.stderr)
+                elif version_obj.status == IDFToolVersion.STATUS_DEPRECATED:
+                    warn('using a deprecated version of tool {} found in PATH: {}'.format(name, tool.version_in_path))
+                continue
+
+        self_restart_cmd = '{} {}{}'.format(sys.executable, __file__,
+                                            (' --tools-json ' + args.tools_json) if args.tools_json else '')
+        self_restart_cmd = to_shell_specific_paths([self_restart_cmd])[0]
+
+        if IDF_TOOLS_EXPORT_CMD:
+            prefer_system_hint = ''
+        else:
+            prefer_system_hint = ' To use it, run \'{} export --prefer-system\''.format(self_restart_cmd)
+
+        if IDF_TOOLS_INSTALL_CMD:
+            install_cmd = to_shell_specific_paths([IDF_TOOLS_INSTALL_CMD])[0]
+        else:
+            install_cmd = self_restart_cmd + ' install'
+
+        if not tool.versions_installed:
+            if tool.get_install_type() == IDFTool.INSTALL_ALWAYS:
+                all_tools_found = False
+                fatal('tool {} has no installed versions. Please run \'{}\' to install it.'.format(
+                    tool.name, install_cmd))
+                if tool.version_in_path and tool.version_in_path not in tool.versions:
+                    info('An unsupported version of tool {} was found in PATH: {}. '.format(name, tool.version_in_path) +
+                         prefer_system_hint, f=sys.stderr)
+                continue
+            else:
+                # tool is optional, and does not have versions installed
+                # use whatever is available in PATH
+                continue
+
+        if tool.version_in_path and tool.version_in_path not in tool.versions:
+            info('Not using an unsupported version of tool {} found in PATH: {}.'.format(
+                 tool.name, tool.version_in_path) + prefer_system_hint, f=sys.stderr)
+
+        version_to_use = tool.get_preferred_installed_version()
+        export_paths = tool.get_export_paths(version_to_use)
+        if export_paths:
+            paths_to_export += export_paths
+        tool_export_vars = tool.get_export_vars(version_to_use)
+        for k, v in tool_export_vars.items():
+            old_v = os.environ.get(k)
+            if old_v is None or old_v != v:
+                export_vars[k] = v
+
+    current_path = os.getenv('PATH')
+    idf_python_env_path, idf_python_export_path, virtualenv_python = get_python_env_path()
+    if os.path.exists(virtualenv_python):
+        idf_python_env_path = to_shell_specific_paths([idf_python_env_path])[0]
+        if os.getenv('IDF_PYTHON_ENV_PATH') != idf_python_env_path:
+            export_vars['IDF_PYTHON_ENV_PATH'] = to_shell_specific_paths([idf_python_env_path])[0]
+        if idf_python_export_path not in current_path:
+            paths_to_export.append(idf_python_export_path)
+
+    idf_tools_dir = os.path.join(global_idf_path, 'tools')
+    idf_tools_dir = to_shell_specific_paths([idf_tools_dir])[0]
+    if idf_tools_dir not in current_path:
+        paths_to_export.append(idf_tools_dir)
+
+    if sys.platform == 'win32' and 'MSYSTEM' not in os.environ:
+        old_path = '%PATH%'
+        path_sep = ';'
+    else:
+        old_path = '$PATH'
+        # can't trust os.pathsep here, since for Windows Python started from MSYS shell,
+        # os.pathsep will be ';'
+        path_sep = ':'
+
+    if args.format == EXPORT_SHELL:
+        if sys.platform == 'win32' and 'MSYSTEM' not in os.environ:
+            export_format = 'SET "{}={}"'
+            export_sep = '\n'
+        else:
+            export_format = 'export {}="{}"'
+            export_sep = ';'
+    elif args.format == EXPORT_KEY_VALUE:
+        export_format = '{}={}'
+        export_sep = '\n'
+    else:
+        raise NotImplementedError('unsupported export format {}'.format(args.format))
+
+    if paths_to_export:
+        export_vars['PATH'] = path_sep.join(to_shell_specific_paths(paths_to_export) + [old_path])
+
+    export_statements = export_sep.join([export_format.format(k, v) for k, v in export_vars.items()])
+
+    if export_statements:
+        print(export_statements)
+
+    if not all_tools_found:
+        raise SystemExit(1)
+
+
+def apply_mirror_prefix_map(args, tool_obj, tool_version):
+    """Rewrite URL for given tool_obj, given tool_version, and current platform,
+       if --mirror-prefix-map flag or IDF_MIRROR_PREFIX_MAP environment variable is given.
+    """
+    mirror_prefix_map = None
+    mirror_prefix_map_env = os.getenv('IDF_MIRROR_PREFIX_MAP')
+    if mirror_prefix_map_env:
+        mirror_prefix_map = mirror_prefix_map_env.split(';')
+    if IDF_MAINTAINER and args.mirror_prefix_map:
+        if mirror_prefix_map:
+            warn('Both IDF_MIRROR_PREFIX_MAP environment variable and --mirror-prefix-map flag are specified, ' +
+                 'will use the value from the command line.')
+        mirror_prefix_map = args.mirror_prefix_map
+    download_obj = tool_obj.versions[tool_version].get_download_for_platform(PYTHON_PLATFORM)
+    if mirror_prefix_map and download_obj:
+        for item in mirror_prefix_map:
+            if URL_PREFIX_MAP_SEPARATOR not in item:
+                warn('invalid mirror-prefix-map item (missing \'{}\') {}'.format(URL_PREFIX_MAP_SEPARATOR, item))
+                continue
+            search, replace = item.split(URL_PREFIX_MAP_SEPARATOR, 1)
+            old_url = download_obj.url
+            new_url = re.sub(search, replace, old_url)
+            if new_url != old_url:
+                info('Changed download URL: {} => {}'.format(old_url, new_url))
+                download_obj.url = new_url
+                break
+
+
+def action_install(args):
+    tools_info = load_tools_info()
+    tools_spec = args.tools
+    if not tools_spec:
+        tools_spec = [k for k, v in tools_info.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
+        info('Installing tools: {}'.format(', '.join(tools_spec)))
+    elif 'all' in tools_spec:
+        tools_spec = [k for k, v in tools_info.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
+        info('Installing tools: {}'.format(', '.join(tools_spec)))
+
+    for tool_spec in tools_spec:
+        if '@' not in tool_spec:
+            tool_name = tool_spec
+            tool_version = None
+        else:
+            tool_name, tool_version = tool_spec.split('@', 1)
+        if tool_name not in tools_info:
+            fatal('unknown tool name: {}'.format(tool_name))
+            raise SystemExit(1)
+        tool_obj = tools_info[tool_name]
+        if tool_version is not None and tool_version not in tool_obj.versions:
+            fatal('unknown version for tool {}: {}'.format(tool_name, tool_version))
+            raise SystemExit(1)
+        if tool_version is None:
+            tool_version = tool_obj.get_recommended_version()
+        assert tool_version is not None
+        tool_obj.find_installed_versions()
+        tool_spec = '{}@{}'.format(tool_name, tool_version)
+        if tool_version in tool_obj.versions_installed:
+            info('Skipping {} (already installed)'.format(tool_spec))
+            continue
+
+        info('Installing {}'.format(tool_spec))
+        apply_mirror_prefix_map(args, tool_obj, tool_version)
+
+        tool_obj.download(tool_version)
+        tool_obj.install(tool_version)
+
+
+def action_install_python_env(args):
+    idf_python_env_path, _, virtualenv_python = get_python_env_path()
+
+    if args.reinstall and os.path.exists(idf_python_env_path):
+        warn('Removing the existing Python environment in {}'.format(idf_python_env_path))
+        shutil.rmtree(idf_python_env_path)
+
+    if not os.path.exists(virtualenv_python):
+        info('Creating a new Python environment in {}'.format(idf_python_env_path))
+
+        try:
+            import virtualenv   # noqa: F401
+        except ImportError:
+            info('Installing virtualenv')
+            subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'virtualenv'],
+                                  stdout=sys.stdout, stderr=sys.stderr)
+
+        subprocess.check_call([sys.executable, '-m', 'virtualenv', '--no-site-packages', idf_python_env_path],
+                              stdout=sys.stdout, stderr=sys.stderr)
+    run_args = [virtualenv_python, '-m', 'pip', 'install', '--no-warn-script-location']
+    requirements_txt = os.path.join(global_idf_path, 'requirements.txt')
+    run_args += ['-r', requirements_txt]
+    if args.extra_wheels_dir:
+        run_args += ['--find-links', args.extra_wheels_dir]
+    info('Installing Python packages from {}'.format(requirements_txt))
+    subprocess.check_call(run_args, stdout=sys.stdout, stderr=sys.stderr)
+
+
+def action_add_version(args):
+    tools_info = load_tools_info()
+    tool_name = args.tool
+    tool_obj = tools_info.get(tool_name)
+    if not tool_obj:
+        info('Creating new tool entry for {}'.format(tool_name))
+        tool_obj = IDFTool(tool_name, TODO_MESSAGE, IDFTool.INSTALL_ALWAYS,
+                           TODO_MESSAGE, TODO_MESSAGE, [TODO_MESSAGE], TODO_MESSAGE)
+        tools_info[tool_name] = tool_obj
+    version = args.version
+    version_obj = tool_obj.versions.get(version)
+    if version not in tool_obj.versions:
+        info('Creating new version {}'.format(version))
+        version_obj = IDFToolVersion(version, IDFToolVersion.STATUS_SUPPORTED)
+        tool_obj.versions[version] = version_obj
+    url_prefix = args.url_prefix or 'https://%s/' % TODO_MESSAGE
+    for file_path in args.files:
+        file_name = os.path.basename(file_path)
+        # Guess which platform this file is for
+        found_platform = None
+        for platform_alias, platform_id in PLATFORM_FROM_NAME.items():
+            if platform_alias in file_name:
+                found_platform = platform_id
+                break
+        if found_platform is None:
+            info('Could not guess platform for file {}'.format(file_name))
+            found_platform = TODO_MESSAGE
+        # Get file size and calculate the SHA256
+        file_size, file_sha256 = get_file_size_sha256(file_path)
+        url = url_prefix + file_name
+        info('Adding download for platform {}'.format(found_platform))
+        info('    size:   {}'.format(file_size))
+        info('    SHA256: {}'.format(file_sha256))
+        info('    URL:    {}'.format(url))
+        version_obj.add_download(found_platform, url, file_size, file_sha256)
+    json_str = dump_tools_json(tools_info)
+    if not args.output:
+        args.output = os.path.join(global_idf_path, TOOLS_FILE_NEW)
+    with open(args.output, 'w') as f:
+        f.write(json_str)
+        f.write('\n')
+    info('Wrote output to {}'.format(args.output))
+
+
+def action_rewrite(args):
+    tools_info = load_tools_info()
+    json_str = dump_tools_json(tools_info)
+    if not args.output:
+        args.output = os.path.join(global_idf_path, TOOLS_FILE_NEW)
+    with open(args.output, 'w') as f:
+        f.write(json_str)
+        f.write('\n')
+    info('Wrote output to {}'.format(args.output))
+
+
+def action_validate(args):
+    try:
+        import jsonschema
+    except ImportError:
+        fatal('You need to install jsonschema package to use validate command')
+        raise SystemExit(1)
+
+    with open(os.path.join(global_idf_path, TOOLS_FILE), 'r') as tools_file:
+        tools_json = json.load(tools_file)
+
+    with open(os.path.join(global_idf_path, TOOLS_SCHEMA_FILE), 'r') as schema_file:
+        schema_json = json.load(schema_file)
+    jsonschema.validate(tools_json, schema_json)
+    # on failure, this will raise an exception with a fairly verbose diagnostic message
+
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--quiet', help='Don\'t output diagnostic messages to stdout/stderr', action='store_true')
+    parser.add_argument('--tools-json', help='Path to the tools.json file to use')
+    parser.add_argument('--idf-path', help='ESP-IDF path to use')
+
+    subparsers = parser.add_subparsers(dest='action')
+    subparsers.add_parser('list', help='List tools and versions available')
+    subparsers.add_parser('check', help='Print summary of tools installed or found in PATH')
+    export = subparsers.add_parser('export', help='Output command for setting tool paths, suitable for shell')
+    export.add_argument('--format', choices=[EXPORT_SHELL, EXPORT_KEY_VALUE], default=EXPORT_SHELL,
+                        help='Format of the output: shell (suitable for printing into shell), ' +
+                             'or key-value (suitable for parsing by other tools')
+    export.add_argument('--prefer-system', help='Normally, if the tool is already present in PATH, ' +
+                                                'but has an unsupported version, a version from the tools directory ' +
+                                                'will be used instead. If this flag is given, the version in PATH ' +
+                                                'will be used.', action='store_true')
+    install = subparsers.add_parser('install', help='Download and install tools into the tools directory')
+    if IDF_MAINTAINER:
+        install.add_argument('--mirror-prefix-map', nargs='*',
+                             help='Pattern to rewrite download URLs, with source and replacement separated by comma.' +
+                                  ' E.g. http://foo.com,http://test.foo.com')
+    install.add_argument('tools', nargs='*', help='Tools to install. ' +
+                                                  'To install a specific version use tool_name@version syntax.' +
+                                                  'Use \'all\' to install all tools, including the optional ones.')
+
+    install_python_env = subparsers.add_parser('install-python-env',
+                                               help='Create Python virtual environment and install the ' +
+                                                    'required Python packages')
+    install_python_env.add_argument('--reinstall', help='Discard the previously installed environment',
+                                    action='store_true')
+    install_python_env.add_argument('--extra-wheels-dir', help='Additional directories with wheels ' +
+                                    'to use during installation')
+
+    if IDF_MAINTAINER:
+        add_version = subparsers.add_parser('add-version', help='Add or update download info for a version')
+        add_version.add_argument('--output', help='Save new tools.json into this file')
+        add_version.add_argument('--tool', help='Tool name to set add a version for', required=True)
+        add_version.add_argument('--version', help='Version identifier', required=True)
+        add_version.add_argument('--url-prefix', help='String to prepend to file names to obtain download URLs')
+        add_version.add_argument('files', help='File names of the download artifacts', nargs='*')
+
+        rewrite = subparsers.add_parser('rewrite', help='Load tools.json, validate, and save the result back into JSON')
+        rewrite.add_argument('--output', help='Save new tools.json into this file')
+
+        subparsers.add_parser('validate', help='Validate tools.json against schema file')
+
+    args = parser.parse_args(argv)
+
+    if args.action is None:
+        parser.print_help()
+        parser.exit(1)
+
+    if args.quiet:
+        global global_quiet
+        global_quiet = True
+
+    global global_idf_path
+    global_idf_path = os.environ.get('IDF_PATH')
+    if args.idf_path:
+        global_idf_path = args.idf_path
+    if not global_idf_path:
+        global_idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+
+    global global_idf_tools_path
+    global_idf_tools_path = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(IDF_TOOLS_PATH_DEFAULT)
+
+    if sys.version_info.major == 2:
+        try:
+            global_idf_tools_path.decode('ascii')
+        except UnicodeDecodeError:
+            fatal('IDF_TOOLS_PATH contains non-ASCII characters: {}'.format(global_idf_tools_path) +
+                  '\nThis is not supported yet with Python 2. ' +
+                  'Please set IDF_TOOLS_PATH to a directory with an ASCII name, or switch to Python 3.')
+            raise SystemExit(1)
+
+    if CURRENT_PLATFORM == UNKNOWN_PLATFORM:
+        fatal('Platform {} appears to be unsupported'.format(PYTHON_PLATFORM))
+        raise SystemExit(1)
+
+    global global_tools_json
+    if args.tools_json:
+        global_tools_json = args.tools_json
+    else:
+        global_tools_json = os.path.join(global_idf_path, TOOLS_FILE)
+
+    action_func_name = 'action_' + args.action.replace('-', '_')
+    action_func = globals()[action_func_name]
+
+    action_func(args)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/tools/test_idf_tools/test_idf_tools.py b/tools/test_idf_tools/test_idf_tools.py
new file mode 100755 (executable)
index 0000000..6e39f5f
--- /dev/null
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+#
+# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import unittest
+import tempfile
+import shutil
+
+try:
+    from contextlib import redirect_stdout
+except ImportError:
+    import contextlib
+
+    @contextlib.contextmanager
+    def redirect_stdout(target):
+        original = sys.stdout
+        sys.stdout = target
+        yield
+        sys.stdout = original
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+# Need to do this before importing idf_tools.py
+os.environ['IDF_MAINTAINER'] = '1'
+
+try:
+    import idf_tools
+except ImportError:
+    sys.path.append('..')
+    import idf_tools
+
+
+class TestUsage(unittest.TestCase):
+
+    def test_usage_basic(self):
+        old_tools_dir = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT)
+
+        mirror_prefix_map = None
+        if os.path.exists(old_tools_dir):
+            mirror_prefix_map = 'https://dl.espressif.com/dl,file://' + os.path.join(old_tools_dir, 'dist')
+            mirror_prefix_map += ';https://github.com/espressif/.*/releases/download/.*/,file://' + os.path.join(old_tools_dir, 'dist', '')
+        if mirror_prefix_map:
+            print('Using IDF_MIRROR_PREFIX_MAP={}'.format(mirror_prefix_map))
+            os.environ['IDF_MIRROR_PREFIX_MAP'] = mirror_prefix_map
+
+        temp_tools_dir = tempfile.mkdtemp(prefix='idf_tools_tmp')
+        print('Using IDF_TOOLS_PATH={}'.format(temp_tools_dir))
+        os.environ['IDF_TOOLS_PATH'] = temp_tools_dir
+
+        self.addCleanup(shutil.rmtree, temp_tools_dir)
+
+        output_stream = StringIO()
+        with redirect_stdout(output_stream):
+            idf_tools.main(['list'])
+        output = output_stream.getvalue()
+
+        xtensa_esp32_elf_version = '1.22.0-80-g6c4433a-5.2.0'
+        esp32ulp_version = '2.28.51.20170517'
+
+        self.assertIn('* xtensa-esp32-elf:', output)
+        self.assertIn('- %s (recommended)' % xtensa_esp32_elf_version, output)
+        self.assertIn('* esp32ulp-elf', output)
+        self.assertIn('- %s (recommended)' % esp32ulp_version, output)
+
+        output_stream = StringIO()
+        with redirect_stdout(output_stream):
+            idf_tools.main(['install'])
+        output = output_stream.getvalue()
+
+        self.assertIn('Installing esp32ulp-elf@' + esp32ulp_version, output)
+        self.assertIn('Downloading binutils-esp32ulp', output)
+        self.assertIn('Installing xtensa-esp32-elf@' + xtensa_esp32_elf_version, output)
+        self.assertIn('Downloading xtensa-esp32-elf', output)
+        self.assertIn('to ' + os.path.join(temp_tools_dir, 'dist'), output)
+
+        output_stream = StringIO()
+        with redirect_stdout(output_stream):
+            idf_tools.main(['check'])
+        output = output_stream.getvalue()
+
+        self.assertIn('version installed in tools directory: ' + esp32ulp_version, output)
+        self.assertIn('version installed in tools directory: ' + xtensa_esp32_elf_version, output)
+
+        output_stream = StringIO()
+        with redirect_stdout(output_stream):
+            idf_tools.main(['export'])
+        output = output_stream.getvalue()
+
+        self.assertIn('%s/tools/esp32ulp-elf/%s/esp32ulp-elf-binutils/bin' %
+                      (temp_tools_dir, esp32ulp_version), output)
+        self.assertIn('%s/tools/xtensa-esp32-elf/%s/xtensa-esp32-elf/bin' %
+                      (temp_tools_dir, xtensa_esp32_elf_version), output)
+
+
+class TestMaintainer(unittest.TestCase):
+
+    def test_validation(self):
+        idf_tools.main(['validate'])
+
+    def test_json_rewrite(self):
+        idf_tools.main(['rewrite'])
+        idf_path = os.getenv('IDF_PATH')
+        if not idf_path:
+            self.fail('IDF_PATH needs to be set to run this test')
+        with open(os.path.join(idf_path, 'tools/tools.json'), 'r') as f:
+            json_old = f.read()
+        with open(os.path.join(idf_path, 'tools/tools.new.json'), 'r') as f:
+            json_new = f.read()
+        self.assertEqual(json_old, json_new)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tools/tools.json b/tools/tools.json
new file mode 100644 (file)
index 0000000..b97eb5e
--- /dev/null
@@ -0,0 +1,347 @@
+{
+  "tools": [
+    {
+      "description": "Toolchain for Xtensa (ESP32) based on GCC",
+      "export_paths": [
+        [
+          "xtensa-esp32-elf",
+          "bin"
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/espressif/crosstool-NG",
+      "install": "always",
+      "license": "GPL-3.0-with-GCC-exception",
+      "name": "xtensa-esp32-elf",
+      "version_cmd": [
+        "xtensa-esp32-elf-gcc",
+        "--version"
+      ],
+      "version_regex": "\\(crosstool-NG\\s+(?:crosstool-ng-)?([0-9a-z\\.\\-]+)\\)\\s*([0-9\\.]+)",
+      "version_regex_replace": "\\1-\\2",
+      "versions": [
+        {
+          "name": "1.22.0-80-g6c4433a5-5.2.0",
+          "status": "recommended",
+          "win32": {
+            "sha256": "f217fccbeaaa8c92db239036e0d6202458de4488b954a3a38f35ac2ec48058a4",
+            "size": 125719261,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
+          },
+          "win64": {
+            "sha256": "f217fccbeaaa8c92db239036e0d6202458de4488b954a3a38f35ac2ec48058a4",
+            "size": 125719261,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
+          }
+        },
+        {
+          "linux-amd64": {
+            "sha256": "3fe96c151d46c1d4e5edc6ed690851b8e53634041114bad04729bc16b0445156",
+            "size": 44219107,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22.0-80-g6c4433a-5.2.0.tar.gz"
+          },
+          "linux-i686": {
+            "sha256": "b4055695ffc2dfc0bcb6dafdc2572a6e01151c4179ef5fa972b3fcb2183eb155",
+            "size": 45566336,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-linux32-1.22.0-80-g6c4433a-5.2.0.tar.gz"
+          },
+          "macos": {
+            "sha256": "a4307a97945d2f2f2745f415fbe80d727750e19f91f9a1e7e2f8a6065652f9da",
+            "size": 46517409,
+            "url": "https://dl.espressif.com/dl/xtensa-esp32-elf-osx-1.22.0-80-g6c4433a-5.2.0.tar.gz"
+          },
+          "name": "1.22.0-80-g6c4433a-5.2.0",
+          "status": "recommended"
+        }
+      ]
+    },
+    {
+      "description": "Toolchain for ESP32 ULP coprocessor",
+      "export_paths": [
+        [
+          "esp32ulp-elf-binutils",
+          "bin"
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/espressif/binutils-esp32ulp",
+      "install": "always",
+      "license": "GPL-2.0-or-later",
+      "name": "esp32ulp-elf",
+      "version_cmd": [
+        "esp32ulp-elf-as",
+        "--version"
+      ],
+      "version_regex": "\\(GNU Binutils\\)\\s+([0-9a-z\\.\\-]+)",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "c1bbcd65e1e30c7312a50344c8dbc70c2941580a79aa8f8abbce8e0e90c79566",
+            "size": 8246604,
+            "url": "https://dl.espressif.com/dl/binutils-esp32ulp-linux64-2.28.51-esp32ulp-20180809.tar.gz"
+          },
+          "macos": {
+            "sha256": "c92937d85cc9a90eb6c6099ce767ca021108c18c94e34bd7b1fa0cde168f94a0",
+            "size": 5726662,
+            "url": "https://dl.espressif.com/dl/binutils-esp32ulp-macos-2.28.51-esp32ulp-20180809.tar.gz"
+          },
+          "name": "2.28.51.20170517",
+          "status": "recommended",
+          "win32": {
+            "sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd",
+            "size": 12231559,
+            "url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip"
+          },
+          "win64": {
+            "sha256": "92dc83e69e534c9f73d7b939088f2e84f757d2478483415d17fe9dd1c236f2fd",
+            "size": 12231559,
+            "url": "https://dl.espressif.com/dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "CMake build system",
+      "export_paths": [
+        [
+          "bin"
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/Kitware/CMake",
+      "install": "on_request",
+      "license": "BSD-3-Clause",
+      "name": "cmake",
+      "platform_overrides": [
+        {
+          "install": "always",
+          "platforms": [
+            "win32",
+            "win64"
+          ]
+        },
+        {
+          "export_paths": [
+            [
+              "CMake.app",
+              "Contents",
+              "bin"
+            ]
+          ],
+          "platforms": [
+            "macos"
+          ]
+        }
+      ],
+      "strip_container_dirs": 1,
+      "version_cmd": [
+        "cmake",
+        "--version"
+      ],
+      "version_regex": "cmake version ([0-9.]+)",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76",
+            "size": 38405896,
+            "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz"
+          },
+          "macos": {
+            "sha256": "fef537614d73fda848f6168273b6c7ba45f850484533361e7bc50ac1d315f780",
+            "size": 32062124,
+            "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Darwin-x86_64.tar.gz"
+          },
+          "name": "3.13.4",
+          "status": "recommended",
+          "win32": {
+            "sha256": "28daf772f55d817a13ef14e25af2a5569f8326dac66a6aa3cc5208cf1f8e943f",
+            "size": 26385104,
+            "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win32-x86.zip"
+          },
+          "win64": {
+            "sha256": "bcd477d49e4a9400b41213d53450b474beaedb264631693c958ef9affa8e5623",
+            "size": 29696565,
+            "url": "https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-win64-x64.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "OpenOCD for ESP32",
+      "export_paths": [
+        [
+          "openocd-esp32",
+          "bin"
+        ]
+      ],
+      "export_vars": {
+        "OPENOCD_SCRIPTS": "${TOOL_PATH}/openocd-esp32/share/openocd/scripts"
+      },
+      "info_url": "https://github.com/espressif/openocd-esp32",
+      "install": "always",
+      "license": "GPL-2.0-only",
+      "name": "openocd-esp32",
+      "version_cmd": [
+        "openocd",
+        "--version"
+      ],
+      "version_regex": "Open On-Chip Debugger\\s+([a-z0-9.-]+)\\s+",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "e5b5579edffde090e426b4995b346e281843bf84394f8e68c8e41bd1e4c576bd",
+            "size": 1681596,
+            "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-linux64-0.10.0-esp32-20190313.tar.gz"
+          },
+          "macos": {
+            "sha256": "09504eea5aa92646a117f16573c95b34e04b4010791a2f8fefcd2bd8c430f081",
+            "size": 1760536,
+            "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-macos-0.10.0-esp32-20190313.tar.gz"
+          },
+          "name": "v0.10.0-esp32-20190313",
+          "status": "recommended",
+          "win32": {
+            "sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2",
+            "size": 2098513,
+            "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip"
+          },
+          "win64": {
+            "sha256": "b86a7f9f39dfc4d8e289fc819375bbb7a5e9fcb8895805ba2b5faf67b8b25ce2",
+            "size": 2098513,
+            "url": "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20190313/openocd-esp32-win32-0.10.0-esp32-20190313.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "menuconfig tool",
+      "export_paths": [
+        [
+          ""
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/espressif/kconfig-frontends",
+      "install": "never",
+      "license": "GPL-2.0-only",
+      "name": "mconf",
+      "platform_overrides": [
+        {
+          "install": "always",
+          "platforms": [
+            "win32",
+            "win64"
+          ]
+        }
+      ],
+      "strip_container_dirs": 1,
+      "version_cmd": [
+        "mconf-idf",
+        "-v"
+      ],
+      "version_regex": "mconf-idf version mconf-([a-z0-9.-]+)-win32",
+      "versions": [
+        {
+          "name": "v4.6.0.0-idf-20190313",
+          "status": "recommended",
+          "win32": {
+            "sha256": "051bef09c782bc31b737b047808d1b6588b0965101b77dc979af9139773c4b4f",
+            "size": 826167,
+            "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190313/mconf-v4.6.0.0-idf-20190313-win32.zip"
+          },
+          "win64": {
+            "sha256": "051bef09c782bc31b737b047808d1b6588b0965101b77dc979af9139773c4b4f",
+            "size": 826167,
+            "url": "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20190313/mconf-v4.6.0.0-idf-20190313-win32.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "Ninja build system",
+      "export_paths": [
+        [
+          ""
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/ninja-build/ninja",
+      "install": "on_request",
+      "license": "Apache-2.0",
+      "name": "ninja",
+      "platform_overrides": [
+        {
+          "install": "always",
+          "platforms": [
+            "win32",
+            "win64"
+          ]
+        }
+      ],
+      "version_cmd": [
+        "ninja",
+        "--version"
+      ],
+      "version_regex": "([0-9.]+)",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "978fd9e26c2db8d33392c6daef50e9edac0a3db6680710a9f9ad47e01f3e49b7",
+            "size": 85276,
+            "url": "https://dl.espressif.com/dl/ninja-1.9.0-linux64.tar.gz"
+          },
+          "macos": {
+            "sha256": "9504cd1783ef3c242d06330a50d54dc8f838b605f5fc3e892c47254929f7350c",
+            "size": 91457,
+            "url": "https://dl.espressif.com/dl/ninja-1.9.0-osx.tar.gz"
+          },
+          "name": "1.9.0",
+          "status": "recommended",
+          "win64": {
+            "sha256": "2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9",
+            "size": 254497,
+            "url": "https://dl.espressif.com/dl/ninja-1.9.0-win64.zip"
+          }
+        }
+      ]
+    },
+    {
+      "description": "Ccache (compiler cache)",
+      "export_paths": [
+        [
+          ""
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/ccache/ccache",
+      "install": "never",
+      "license": "GPL-3.0-or-later",
+      "name": "ccache",
+      "platform_overrides": [
+        {
+          "install": "always",
+          "platforms": [
+            "win64"
+          ]
+        }
+      ],
+      "version_cmd": [
+        "ccache.exe",
+        "--version"
+      ],
+      "version_regex": "ccache version ([0-9.]+)",
+      "versions": [
+        {
+          "name": "3.7",
+          "status": "recommended",
+          "win64": {
+            "sha256": "37e833f3f354f1145503533e776c1bd44ec2e77ff8a2476a1d2039b0b10c78d6",
+            "size": 142401,
+            "url": "https://dl.espressif.com/dl/ccache-3.7-w64.zip"
+          }
+        }
+      ]
+    }
+  ],
+  "version": 1
+}
diff --git a/tools/tools_schema.json b/tools/tools_schema.json
new file mode 100644 (file)
index 0000000..6f11215
--- /dev/null
@@ -0,0 +1,234 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/espressif/esp-idf/blob/master/tools/tools-schema.json",
+  "type": "object",
+  "properties": {
+    "version": {
+      "type": "integer",
+      "description": "Metadata file version"
+    },
+    "tools": {
+      "type": "array",
+      "description": "List of tools",
+      "items": {
+        "$ref": "#/definitions/toolInfo"
+      }
+    }
+  },
+  "required": [
+    "version",
+    "tools"
+  ],
+  "definitions": {
+    "toolInfo": {
+      "type": "object",
+      "description": "Information about one tool",
+      "properties": {
+        "name" : {
+          "description": "Tool name (used as a directory name)",
+          "type": "string"
+        },
+        "description" : {
+          "description": "A short (one sentence) description of the tool.",
+          "type": "string"
+        },
+        "export_paths": {
+          "$ref": "#/definitions/exportPaths"
+        },
+        "export_vars": {
+          "$ref": "#/definitions/envVars",
+          "description": "Some variable expansions are done on the values. 1) ${TOOL_PATH} is replaced with the directory where the tool is installed."
+        },
+        "info_url": {
+          "description": "URL of the page with information about the tool",
+          "type": "string"
+        },
+        "install": {
+          "$ref": "#/definitions/installRequirementInfo",
+          "description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation."
+        },
+        "license": {
+          "description": "License name. Use SPDX license identifier if it exists, short name of the license otherwise.",
+          "type": "string"
+        },
+        "version_cmd": {
+          "$ref": "#/definitions/arrayOfStrings",
+          "description": "Command to be executed (along with any extra arguments). The executable be present in one of the export_paths."
+        },
+        "version_regex": {
+          "description": "Regex which is to be applied to version_cmd output to extract the version. By default, the version will be the first capture group of the expression. If version_regex_replace is specified, version will be obtained by doing a substitution using version_regex_replace instead.",
+          "$ref": "#/definitions/regex"
+        },
+        "version_regex_replace": {
+          "description": "If given, this will be used as substitute expression for the regex defined in version_regex, to obtain the version string. Not specifying this is equivalent to setting it to '\\1' (i.e. return the first capture group).",
+          "type": "string"
+        },
+        "strip_container_dirs": {
+          "type": "integer",
+          "description": "If specified, this number of top directory levels will removed when extracting. E.g. if strip_container_dirs=2, archive path a/b/c/d.txt will be extracted as c/d.txt"
+        },
+        "versions": {
+          "type": "array",
+          "description": "List of versions",
+          "items": {
+            "$ref": "#/definitions/versionInfo"
+          }
+        },
+        "platform_overrides": {
+          "type": "array",
+          "description": "List of platform-specific overrides",
+          "items": {
+            "$ref": "#/definitions/platformOverrideInfo"
+          }
+        }
+      },
+      "required": [
+        "description",
+        "export_paths",
+        "version_cmd",
+        "version_regex",
+        "versions",
+        "install",
+        "info_url",
+        "license"
+      ]
+    },
+    "arrayOfStrings": {
+      "description": "Array of strings. Used to represent paths (split into components) and command lines (split into arguments)",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "exportPaths": {
+      "description": "Array of paths to be exported (added to PATH). Each item in the array is relative to the directory where the tool will be installed.",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/arrayOfStrings"
+      }
+    },
+    "envVars": {
+      "description": "Collection of environment variables. Keys and values are the environment variable names and values, respectively.",
+      "type": "object",
+      "patternProperties": {
+        "^([A-Z_0-9]+)+$": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false
+    },
+    "regex": {
+      "description": "A regular expression",
+      "type": "string"
+    },
+    "versionInfo": {
+      "type": "object",
+      "properties": {
+        "name" : {
+          "description": "Version name (used as a directory name)",
+          "type": "string"
+        },
+        "status": {
+            "description": "Determines whether the version is recommended/supported/deprecated",
+            "type": "string",
+            "enum": ["recommended", "supported", "deprecated"]
+        },
+        "linux-i686": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "linux-amd64": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "linux-armel": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "linux-arm64": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "macos": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "win32": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "win64": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        },
+        "any": {
+          "$ref": "#/definitions/platformDownloadInfo"
+        }
+      }
+    },
+    "platformDownloadInfo": {
+      "description": "Information about download artifact for one platform",
+      "type": "object",
+      "properties": {
+        "sha256": {
+          "type": "string",
+          "description": "SHA256 sum of the file"
+        },
+        "size": {
+          "type": "integer",
+          "description": "Size of the file, in bytes"
+        },
+        "url": {
+          "type": "string",
+          "description": "Download URL"
+        }
+      },
+      "required": [
+        "sha256",
+        "url",
+        "size"
+      ]
+    },
+    "installRequirementInfo": {
+      "description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation.",
+      "type": "string",
+      "enum": ["always", "on_request", "never"]
+    },
+    "platformOverrideInfo": {
+      "description": "Platform-specific values which override the defaults",
+      "type": "object",
+      "properties": {
+        "platforms": {
+          "description": "List of platforms to which this override applies",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "enum": ["linux-i686", "linux-amd64", "linux-armel", "linux-arm64", "macos", "win32", "win64"]
+          }
+        },
+        "export_paths": {
+          "description": "Platform-specific replacement for toolInfo/export_paths",
+          "$ref": "#/definitions/exportPaths"
+        },
+        "export_vars": {
+          "description": "Platform-specific replacement for toolInfo/export_vars",
+          "$ref": "#/definitions/envVars"
+        },
+        "install": {
+          "description": "Platform-specific replacement for toolInfo/install",
+          "$ref": "#/definitions/installRequirementInfo"
+        },
+        "version_cmd": {
+          "description": "Platform-specific replacement for toolInfo/version_cmd",
+          "$ref": "#/definitions/arrayOfStrings"
+        },
+        "version_regex": {
+          "description": "Platform-specific replacement for toolInfo/version_regex",
+          "$ref": "#/definitions/regex"
+        },
+        "version_regex_replace": {
+          "description": "Platform-specific replacement for toolInfo/version_regex_replace",
+          "type": "string"
+        },
+        "strip_container_dirs": {
+          "type": "string",
+          "description": "Platform-specific replacement for toolInfo/strip_container_dirs"
+        }
+      },
+      "required": ["platforms"]
+    }
+  }
+}