Docs: Check Python packages

diff --git a/docs/conf_common.py b/docs/conf_common.py
index 244810a9fbe68177bd807a7433035f750c2e4b6c..4ac06b7f78c0c4599502ce9b7188c8e190fd7bba 100644 (file)
--- a/docs/conf_common.py
+++ b/docs/conf_common.py
@@ -30,6 +30,11 @@ builddir = builddir
 if 'BUILDDIR' in os.environ:
     builddir = os.environ['BUILDDIR']
 
+def call_with_python(cmd):
+    # using sys.executable ensures that the scripts are called with the same Python interpreter
+    if os.system('{} {}'.format(sys.executable, cmd)) != 0:
+        raise RuntimeError('{} failed'.format(cmd))
+
 # Call Doxygen to get XML files from the header files
 print("Calling Doxygen to generate latest XML files")
 if os.system("doxygen ../Doxyfile") != 0:
@@ -40,20 +45,17 @@ if os.system("doxygen ../Doxyfile") != 0:
 copy_if_modified('xml/', 'xml_in/')
 
 # Generate 'api_name.inc' files using the XML files by Doxygen
-if os.system('python ../gen-dxd.py') != 0:
-    raise RuntimeError('gen-dxd.py failed')
+call_with_python('../gen-dxd.py')
 
 # Generate 'kconfig.inc' file from components' Kconfig files
 kconfig_inc_path = '{}/inc/kconfig.inc'.format(builddir)
-if os.system('python ../gen-kconfig-doc.py > ' + kconfig_inc_path + '.in') != 0:
-    raise RuntimeError('gen-kconfig-doc.py failed')
+call_with_python('../gen-kconfig-doc.py > ' + kconfig_inc_path + '.in')
 
 copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path)
 
 # Generate 'esp_err_defs.inc' file with ESP_ERR_ error code definitions
 esp_err_inc_path = '{}/inc/esp_err_defs.inc'.format(builddir)
-if os.system('python ../../tools/gen_esp_err_to_name.py --rst_output ' + esp_err_inc_path + '.in') != 0:
-    raise RuntimeError('gen_esp_err_to_name.py failed')
+call_with_python('../../tools/gen_esp_err_to_name.py --rst_output ' + esp_err_inc_path + '.in')
 copy_if_modified(esp_err_inc_path + '.in', esp_err_inc_path)
 
 # Generate version-related includes
@@ -62,8 +64,7 @@ copy_if_modified(esp_err_inc_path + '.in', esp_err_inc_path)
 def generate_version_specific_includes(app):
     print("Generating version-specific includes...")
     version_tmpdir = '{}/version_inc'.format(builddir)
-    if os.system('python ../gen-version-specific-includes.py {} {}'.format(app.config.language, version_tmpdir)):
-        raise RuntimeError('gen-version-specific-includes.py failed')
+    call_with_python('../gen-version-specific-includes.py {} {}'.format(app.config.language, version_tmpdir))
     copy_if_modified(version_tmpdir, '{}/inc'.format(builddir))
 
 

diff --git a/docs/docs_common.mk b/docs/docs_common.mk
index c79cb0d4b6bd0612827e57031992e1f9f3d423f5..d4d623ec8963e5a3c917101072dd4b40d05dd965 100644 (file)
--- a/docs/docs_common.mk
+++ b/docs/docs_common.mk
@@ -28,7 +28,7 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) -w
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext dependencies version-specific-includes
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext dependencies version-specific-includes check_python_packages
 
 help:
        @echo "Please use \`make <target>\' where <target> is one of"
@@ -58,38 +58,44 @@ help:
 clean:
        rm -rf $(BUILDDIR)/*
 
-html:
+# Notify users when some of the required python packages are not installed.
+# Note: This is intended to help developers who generate the documentation on their local machine. Read The Docs uses
+# the requirements.txt file directly and calls sphinx also directly without the use of the makefile!
+check_python_packages:
+       $(IDF_PATH)/tools/check_python_dependencies.py -r $(IDF_PATH)/docs/requirements.txt
+
+html: | check_python_packages
        $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
        @echo
        @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-dirhtml:
+dirhtml: | check_python_packages
        $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
        @echo
        @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
-singlehtml:
+singlehtml: | check_python_packages
        $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
        @echo
        @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 
-pickle:
+pickle: | check_python_packages
        $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
        @echo
        @echo "Build finished; now you can process the pickle files."
 
-json:
+json: | check_python_packages
        $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
        @echo
        @echo "Build finished; now you can process the JSON files."
 
-htmlhelp:
+htmlhelp: | check_python_packages
        $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
        @echo
        @echo "Build finished; now you can run HTML Help Workshop with the" \
              ".hhp project file in $(BUILDDIR)/htmlhelp."
 
-qthelp:
+qthelp: | check_python_packages
        $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
        @echo
        @echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -98,7 +104,7 @@ qthelp:
        @echo "To view the help file:"
        @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ReadtheDocsTemplate.qhc"
 
-devhelp:
+devhelp: | check_python_packages
        $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
        @echo
        @echo "Build finished."
@@ -107,70 +113,70 @@ devhelp:
        @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ReadtheDocsTemplate"
        @echo "# devhelp"
 
-epub:
+epub: | check_python_packages
        $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
        @echo
        @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 
-latex:
+latex: | check_python_packages
        $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
        @echo
        @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
        @echo "Run \`make' in that directory to run these through (pdf)latex" \
              "(use \`make latexpdf' here to do that automatically)."
 
-latexpdf:
+latexpdf: | check_python_packages
        $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
        @echo "Running LaTeX files through pdflatex..."
        $(MAKE) -C $(BUILDDIR)/latex all-pdf
        @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-latexpdfja:
+latexpdfja: | check_python_packages
        $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
        @echo "Running LaTeX files through platex and dvipdfmx..."
        $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
        @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-text:
+text: | check_python_packages
        $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
        @echo
        @echo "Build finished. The text files are in $(BUILDDIR)/text."
 
-man:
+man: | check_python_packages
        $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
        @echo
        @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
 
-texinfo:
+texinfo: | check_python_packages
        $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
        @echo
        @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
        @echo "Run \`make' in that directory to run these through makeinfo" \
              "(use \`make info' here to do that automatically)."
 
-info:
+info: | check_python_packages
        $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
        @echo "Running Texinfo files through makeinfo..."
        make -C $(BUILDDIR)/texinfo info
        @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
 
-gettext:
+gettext: | check_python_packages
        $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
        @echo
        @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
-changes:
+changes: | check_python_packages
        $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
        @echo
        @echo "The overview file is in $(BUILDDIR)/changes."
 
-linkcheck:
+linkcheck: | check_python_packages
        $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
        @echo
        @echo "Link check complete; look for any errors in the above output " \
              "or in $(BUILDDIR)/linkcheck/output.txt."
 
-gh-linkcheck:
+gh-linkcheck: | check_python_packages
        @echo "Checking for hardcoded GitHub links"
        @if (find ../ -name '*.rst' | xargs grep \
                'https://github.com/espressif/esp-idf/tree\|https://github.com/espressif/esp-idf/blob\|https://github.com/espressif/esp-idf/raw'\
@@ -194,17 +200,17 @@ gh-linkcheck:
        fi
        @echo "No hardcoded links found"
 
-doctest:
+doctest: | check_python_packages
        $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
        @echo "Testing of doctests in the sources finished, look at the " \
              "results in $(BUILDDIR)/doctest/output.txt."
 
-xml:
+xml: | check_python_packages
        $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
        @echo
        @echo "Build finished. The XML files are in $(BUILDDIR)/xml."
 
-pseudoxml:
+pseudoxml: | check_python_packages
        $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
        @echo
        @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

diff --git a/docs/gen-dxd.py b/docs/gen-dxd.py
old mode 100644 (file)
new mode 100755 (executable)
index 0737d94..f04c0e0
--- a/docs/gen-dxd.py
+++ b/docs/gen-dxd.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+#
 # gen-dxd.py - Generate Doxygen Directives
 #
 # This code is in the Public Domain (or CC0 licensed, at your option.)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index b85fc09a173b0e83936c80dec87b49f71cd3cf10..dfd6f5c08a445ed9612033fcc874ba482f36faf8 100644 (file)
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -4,8 +4,9 @@
 sphinx==1.6.5
 sphinx-rtd-theme
 breathe==4.7.3
-sphinxcontrib.blockdiag==1.5.3
-sphinxcontrib.seqdiag==0.8.5
-sphinxcontrib.actdiag==0.8.5
-sphinxcontrib.nwdiag==0.9.5
+sphinxcontrib-blockdiag==1.5.3
+sphinxcontrib-seqdiag==0.8.5
+sphinxcontrib-actdiag==0.8.5
+sphinxcontrib-nwdiag==0.9.5
 recommonmark
+future>=0.16.0 # for ../tools/gen_esp_err_to_name.py

diff --git a/tools/check_python_dependencies.py b/tools/check_python_dependencies.py
old mode 100644 (file)
new mode 100755 (executable)
index 9c5f8e8..30eaffc
--- a/tools/check_python_dependencies.py
+++ b/tools/check_python_dependencies.py
@@ -16,6 +16,7 @@
 
 import os
 import sys
+import argparse
 try:
     import pkg_resources
 except:
@@ -24,11 +25,17 @@ except:
            'setting up the required packages.')
     sys.exit(1)
 
-req_file = '{}/requirements.txt'.format(os.getenv("IDF_PATH"))
-
 if __name__ == "__main__":
+    idf_path = os.getenv("IDF_PATH")
+
+    parser = argparse.ArgumentParser(description='ESP32 Python package dependency checker')
+    parser.add_argument('--requirements', '-r',
+            help='Path to the requrements file',
+            default=idf_path + '/requirements.txt')
+    args = parser.parse_args()
+
     not_satisfied = []
-    with open(req_file) as f:
+    with open(args.requirements) as f:
         for line in f:
             line = line.strip()
             try:
@@ -40,7 +47,7 @@ if __name__ == "__main__":
         print('The following Python requirements are not satisfied:')
         for requirement in not_satisfied:
             print(requirement)
-        print('Please run "{} -m pip install -r {}" for resolving the issue.'.format(sys.executable, req_file))
+        print('Please run "{} -m pip install -r {}" for resolving the issue.'.format(sys.executable, args.requirements))
         sys.exit(1)
 
-    print('Python requirements are satisfied.')
+    print('Python requirements from {} are satisfied.'.format(args.requirements))

diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt
index 79fcb31dc23a3998c46af8fdbb07f1c50eb9691e..0ad9b5e6a9f11c5495b975a6807cf89bfc8821e4 100644 (file)
--- a/tools/ci/executable-list.txt
+++ b/tools/ci/executable-list.txt
@@ -41,3 +41,5 @@ tools/test_idf_monitor/run_test_idf_monitor.py
 tools/mass_mfg/mfg_gen.py
 tools/unit-test-app/unit_test.py
 tools/test_idf_size/test.sh
+tools/check_python_dependencies.py
+docs/gen-dxd.py

diff --git a/tools/gen_esp_err_to_name.py b/tools/gen_esp_err_to_name.py
index b37054152bd13cc89855104ddb846b34d3d62eff..d623815ecd09b6822edc857f8c9ef1c733651370 100755 (executable)
--- a/tools/gen_esp_err_to_name.py
+++ b/tools/gen_esp_err_to_name.py
@@ -16,9 +16,20 @@
 
 from __future__ import print_function
 from __future__ import unicode_literals
-from builtins import str
-from builtins import range
-from builtins import object
+import sys
+try:
+    from builtins import str
+    from builtins import range
+    from builtins import object
+except ImportError:
+    # This should not happen because the Python packages are checked before invoking this script. However, here is
+    # some output which should help if we missed something.
+    print('Import has failed probably because of the missing "future" package. Please install all the packages for '
+          'interpreter {} from the requirements.txt file.'.format(sys.executable))
+    # The path to requirements.txt is not provided because this script could be invoked from an IDF project (then the
+    # requirements.txt from the IDF_PATH should be used) or from the documentation project (then the requirements.txt
+    # for the documentation directory should be used).
+    sys.exit(1)
 from io import open
 import os
 import argparse