docs: speed up incremental builds

On each documentation build (‘make html’), doxygen regenerates XML
files. In addition to that, gen-dxd.py regenerates API reference
files under _build/inc/. This results in Sphinx flagging about half
of the input files as modified, and incremental builds taking long
time.

With this change, XML files generated by Doxygen are copied into
docs/xml_in directory only when they are changed. Breathe is pointed
to docs/xml_in directory instead of docs/xml. In addition to that,
gen-dxd.py is modified to only write to the output file when contents
change.

Overall, incremental build time (with no source files changed) is
reduced from ~7 minutes to ~8 seconds (on a particular OS X
computer).

Due to the way Breathe includes Doxygen XML files, there is still
going to be a massive rebuild every time functions, enums, macros,
structures are added or removed from the header files scanned
by Doxygen, but at least individual .rst files can be edited
at a much faster pace.

diff --git a/.gitignore b/.gitignore
index 3abace88d7c2d7d1ecf8bcff857c6bb3d70a68b4..2b5352ff21b2b676b625969945b42e61cfc6c0af 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -32,7 +32,9 @@ docs/doxygen-warning-log.txt
 docs/sphinx-warning-log.txt
 docs/sphinx-warning-log-sanitized.txt
 docs/xml/
+docs/xml_in/
 docs/man/
+docs/doxygen_sqlite3.db
 
 # Unit test app files
 tools/unit-test-app/sdkconfig

diff --git a/docs/conf.py b/docs/conf.py
index aee4803d432a50f49a00c3064d34cd33564fd949..a2cafdbf0a482c72e8e11624f6c4f45a5fa6419d 100644 (file)
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -22,15 +22,26 @@ import shlex
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('.'))
 
-from repo_util import run_cmd_get_output
+from local_util import run_cmd_get_output, copy_if_modified
+
+builddir = '_build'
+if 'BUILDDIR' in os.environ:
+    builddir = os.environ['BUILDDIR']
 
 # Call Doxygen to get XML files from the header files
 print "Calling Doxygen to generate latest XML files"
 call('doxygen')
+# Doxygen has generated XML files in 'xml' directory.
+# Copy them to 'xml_in', only touching the files which have changed.
+copy_if_modified('xml/', 'xml_in/')
+
 # Generate 'api_name.inc' files using the XML files by Doxygen
-os.system("python gen-dxd.py")
+os.system('python gen-dxd.py')
+
 # Generate 'kconfig.inc' file from components' Kconfig files
-os.system("python gen-kconfig-doc.py > _build/inc/kconfig.inc")
+kconfig_inc_path = '{}/inc/kconfig.inc'.format(builddir)
+os.system('python gen-kconfig-doc.py > ' + kconfig_inc_path + '.in')
+copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path)
 
 # http://stackoverflow.com/questions/12772927/specifying-an-online-image-in-sphinx-restructuredtext-format
 # 
@@ -63,7 +74,11 @@ rackdiag_fontpath = '_static/DejaVuSans.ttf'
 packetdiag_fontpath = '_static/DejaVuSans.ttf'
 
 # Breathe extension variables
-breathe_projects = { "esp32-idf": "xml/" }
+
+# Doxygen regenerates files in 'xml/' directory every time,
+# but we copy files to 'xml_in/' only when they change, to speed up
+# incremental builds.
+breathe_projects = { "esp32-idf": "xml_in/" }
 breathe_default_project = "esp32-idf"
 
 # Add any paths that contain templates here, relative to this directory.

diff --git a/docs/gen-dxd.py b/docs/gen-dxd.py
index 4be75915b7b1c38eeeec16fc5dffbf0651a0e4d8..4ebf06b43d43cec0838c54a83b0062837dfd3b99 100644 (file)
--- a/docs/gen-dxd.py
+++ b/docs/gen-dxd.py
@@ -10,6 +10,11 @@ import sys
 import os
 import re
 
+# Determime build directory
+builddir = '_build'
+if 'BUILDDIR' in os.environ:
+    builddir = os.environ['BUILDDIR']
+
 # Script configuration
 header_file_path_prefix = "../components/"
 """string: path prefix for header files.
@@ -20,7 +25,7 @@ doxyfile_path = "Doxyfile"
 xml_directory_path = "xml"
 """string: path to directory with XML files by Doxygen.
 """
-inc_directory_path = "_build/inc"
+inc_directory_path = os.path.join(builddir, 'inc')
 """string: path prefix for header files.
 """
 all_kinds = [
@@ -263,9 +268,15 @@ def generate_api_inc_files():
         api_name = get_api_name(header_file_path)
         inc_file_path = inc_directory_path + "/" + api_name + ".inc"
         rst_output = generate_directives(header_file_path)
-        inc_file = open(inc_file_path, "w")
-        inc_file.write(rst_output)
-        inc_file.close()
+
+        previous_rst_output = ''
+        if os.path.isfile(inc_file_path):
+            with open(inc_file_path, "r") as inc_file_old:
+                previous_rst_output = inc_file_old.read()
+
+        if previous_rst_output != rst_output:
+            with open(inc_file_path, "w") as inc_file:
+                inc_file.write(rst_output)
 
 
 if __name__ == "__main__":

diff --git a/docs/link-roles.py b/docs/link-roles.py
index a9e45145ecdbcda63a15d1d21486bdaa47183d92..7bfe353921812b218bfccee5a88cfe8028fa0051 100644 (file)
--- a/docs/link-roles.py
+++ b/docs/link-roles.py
@@ -2,7 +2,7 @@
 
 import re
 from docutils import nodes
-from repo_util import run_cmd_get_output
+from local_util import run_cmd_get_output
 
 def get_github_rev():
     path = run_cmd_get_output('git rev-parse --short HEAD')

diff --git a/docs/local_util.py b/docs/local_util.py
new file mode 100644 (file)
index 0000000..d85ab38
--- /dev/null
+++ b/docs/local_util.py
@@ -0,0 +1,53 @@
+# Utility functions used in conf.py
+#
+# Copyright 2017 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import os
+import shutil
+
+def run_cmd_get_output(cmd):
+    return os.popen(cmd).read().strip()
+
+def files_equal(path_1, path_2):
+    if not os.path.exists(path_1) or not os.path.exists(path_2):
+        return False
+    file_1_contents = ''
+    with open(path_1, "r") as f_1:
+        file_1_contents = f_1.read()
+    file_2_contents = ''
+    with open(path_2, "r") as f_2:
+        file_2_contents = f_2.read()
+    return file_1_contents == file_2_contents
+
+def copy_file_if_modified(src_file_path, dst_file_path):
+    if not files_equal(src_file_path, dst_file_path):
+        dst_dir_name = os.path.dirname(dst_file_path)
+        if not os.path.isdir(dst_dir_name):
+            os.makedirs(dst_dir_name)
+        shutil.copy(src_file_path, dst_file_path)
+
+def copy_if_modified(src_path, dst_path):
+    if os.path.isfile(src_path):
+        copy_file_if_modified(src_path, dst_path)
+        return
+
+    src_path_len = len(src_path)
+    for root, dirs, files in os.walk(src_path):
+        for src_file_name in files:
+            src_file_path = os.path.join(root, src_file_name)
+            dst_file_path = os.path.join(dst_path + root[src_path_len:], src_file_name)
+            copy_file_if_modified(src_file_path, dst_file_path)
+

diff --git a/docs/repo_util.py b/docs/repo_util.py
deleted file mode 100644 (file)
index 6249c11..0000000
--- a/docs/repo_util.py
+++ /dev/null
@@ -1,5 +0,0 @@
-import re
-import os
-
-def run_cmd_get_output(cmd):
-    return os.popen(cmd).read().strip()