3 # 'idf.py' is a top-level config/build command line tool for ESP-IDF
5 # You don't have to use idf.py, you can use cmake directly
6 # (or use cmake in an IDE)
10 # Copyright 2018 Espressif Systems (Shanghai) PTE LTD
12 # Licensed under the Apache License, Version 2.0 (the "License");
13 # you may not use this file except in compliance with the License.
14 # You may obtain a copy of the License at
16 # http://www.apache.org/licenses/LICENSE-2.0
18 # Unless required by applicable law or agreed to in writing, software
19 # distributed under the License is distributed on an "AS IS" BASIS,
20 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
25 # Note: we don't check for Python build-time dependencies until
26 # check_environment() function below. If possible, avoid importing
27 # any external libraries here - put in external script, or import in
28 # their specific function instead.
34 import multiprocessing
38 import serial.tools.list_ports
40 class FatalError(RuntimeError):
42 Wrapper class for runtime errors that aren't caused by bugs in idf.py or the build proces.s
46 # Use this Python interpreter for any subprocesses we launch
49 # note: os.environ changes don't automatically propagate to child processes,
50 # you have to pass env=os.environ explicitly anywhere that we create a process
51 os.environ["PYTHON"]=sys.executable
53 # Make flavors, across the various kinds of Windows environments & POSIX...
54 if "MSYSTEM" in os.environ: # MSYS
56 MAKE_GENERATOR = "MSYS Makefiles"
57 elif os.name == 'nt': # other Windows
58 MAKE_CMD = "mingw32-make"
59 MAKE_GENERATOR = "MinGW Makefiles"
62 MAKE_GENERATOR = "Unix Makefiles"
65 # ('generator name', 'build command line', 'version command line', 'verbose flag')
66 ("Ninja", [ "ninja" ], [ "ninja", "--version" ], "-v"),
67 (MAKE_GENERATOR, [ MAKE_CMD, "-j", str(multiprocessing.cpu_count()+2) ], [ "make", "--version" ], "VERBOSE=1"),
69 GENERATOR_CMDS = dict( (a[0], a[1]) for a in GENERATORS )
70 GENERATOR_VERBOSE = dict( (a[0], a[3]) for a in GENERATORS )
72 def _run_tool(tool_name, args, cwd):
74 " Quote 'arg' if necessary "
75 if " " in arg and not (arg.startswith('"') or arg.startswith("'")):
76 return "'" + arg + "'"
78 display_args = " ".join(quote_arg(arg) for arg in args)
79 print("Running %s in directory %s" % (tool_name, quote_arg(cwd)))
80 print('Executing "%s"...' % display_args)
82 # Note: we explicitly pass in os.environ here, as we may have set IDF_PATH there during startup
83 subprocess.check_call(args, env=os.environ, cwd=cwd)
84 except subprocess.CalledProcessError as e:
85 raise FatalError("%s failed with exit code %d" % (tool_name, e.returncode))
88 def check_environment():
90 Verify the environment contains the top-level tools we need to operate
92 (cmake will check a lot of other things)
94 if not executable_exists(["cmake", "--version"]):
95 raise FatalError("'cmake' must be available on the PATH to use idf.py")
96 # find the directory idf.py is in, then the parent directory of this, and assume this is IDF_PATH
97 detected_idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
98 if "IDF_PATH" in os.environ:
99 set_idf_path = os.path.realpath(os.environ["IDF_PATH"])
100 if set_idf_path != detected_idf_path:
101 print("WARNING: IDF_PATH environment variable is set to %s but idf.py path indicates IDF directory %s. Using the environment variable directory, but results may be unexpected..."
102 % (set_idf_path, detected_idf_path))
104 print("Setting IDF_PATH environment variable: %s" % detected_idf_path)
105 os.environ["IDF_PATH"] = detected_idf_path
107 # check Python dependencies
108 print("Checking Python dependencies...")
110 subprocess.check_call([ os.environ["PYTHON"],
111 os.path.join(os.environ["IDF_PATH"], "tools", "check_python_dependencies.py")],
113 except subprocess.CalledProcessError:
116 def executable_exists(args):
118 subprocess.check_output(args)
123 def detect_cmake_generator():
125 Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found.
127 for (generator, _, version_check, _) in GENERATORS:
128 if executable_exists(version_check):
130 raise FatalError("To use idf.py, either the 'ninja' or 'GNU make' build tool must be available in the PATH")
132 def _ensure_build_directory(args, always_run_cmake=False):
133 """Check the build directory exists and that cmake has been run there.
135 If this isn't the case, create the build directory (if necessary) and
136 do an initial cmake run to configure it.
138 This function will also check args.generator parameter. If the parameter is incompatible with
139 the build directory, an error is raised. If the parameter is None, this function will set it to
140 an auto-detected default generator or to the value already configured in the build directory.
142 project_dir = args.project_dir
143 # Verify the project directory
144 if not os.path.isdir(project_dir):
145 if not os.path.exists(project_dir):
146 raise FatalError("Project directory %s does not exist")
148 raise FatalError("%s must be a project directory")
149 if not os.path.exists(os.path.join(project_dir, "CMakeLists.txt")):
150 raise FatalError("CMakeLists.txt not found in project directory %s" % project_dir)
152 # Verify/create the build directory
153 build_dir = args.build_dir
154 if not os.path.isdir(build_dir):
156 cache_path = os.path.join(build_dir, "CMakeCache.txt")
157 if not os.path.exists(cache_path) or always_run_cmake:
158 if args.generator is None:
159 args.generator = detect_cmake_generator()
161 cmake_args = ["cmake", "-G", args.generator, "-DPYTHON_DEPS_CHECKED=1"]
162 if not args.no_warnings:
163 cmake_args += [ "--warn-uninitialized" ]
165 cmake_args += [ "-DCCACHE_DISABLE=1" ]
166 cmake_args += [ project_dir]
167 _run_tool("cmake", cmake_args, cwd=args.build_dir)
169 # don't allow partially valid CMakeCache.txt files,
170 # to keep the "should I run cmake?" logic simple
171 if os.path.exists(cache_path):
172 os.remove(cache_path)
175 # Learn some things from the CMakeCache.txt file in the build directory
176 cache = parse_cmakecache(cache_path)
178 generator = cache["CMAKE_GENERATOR"]
180 generator = detect_cmake_generator()
181 if args.generator is None:
182 args.generator = generator # reuse the previously configured generator, if none was given
183 if generator != args.generator:
184 raise FatalError("Build is configured for generator '%s' not '%s'. Run 'idf.py fullclean' to start again."
185 % (generator, args.generator))
188 home_dir = cache["CMAKE_HOME_DIRECTORY"]
189 if os.path.normcase(os.path.realpath(home_dir)) != os.path.normcase(os.path.realpath(project_dir)):
190 raise FatalError("Build directory '%s' configured for project '%s' not '%s'. Run 'idf.py fullclean' to start again."
191 % (build_dir, os.path.realpath(home_dir), os.path.realpath(project_dir)))
193 pass # if cmake failed part way, CMAKE_HOME_DIRECTORY may not be set yet
196 def parse_cmakecache(path):
198 Parse the CMakeCache file at 'path'.
200 Returns a dict of name:value.
202 CMakeCache entries also each have a "type", but this is currently ignored.
205 with open(path) as f:
207 # cmake cache lines look like: CMAKE_CXX_FLAGS_DEBUG:STRING=-g
208 # groups are name, type, value
209 m = re.match(r"^([^#/:=]+):([^:=]+)=(.+)\n$", line)
211 result[m.group(1)] = m.group(3)
214 def build_target(target_name, args):
216 Execute the target build system to build target 'target_name'
218 Calls _ensure_build_directory() which will run cmake to generate a build
219 directory (with the specified generator) as needed.
221 _ensure_build_directory(args)
222 generator_cmd = GENERATOR_CMDS[args.generator]
223 if not args.no_ccache:
224 # Setting CCACHE_BASEDIR & CCACHE_NO_HASHDIR ensures that project paths aren't stored in the ccache entries
225 # (this means ccache hits can be shared between different projects. It may mean that some debug information
226 # will point to files in another project, if these files are perfect duplicates of each other.)
228 # It would be nicer to set these from cmake, but there's no cross-platform way to set build-time environment
229 #os.environ["CCACHE_BASEDIR"] = args.build_dir
230 #os.environ["CCACHE_NO_HASHDIR"] = "1"
233 generator_cmd += [ GENERATOR_VERBOSE[args.generator] ]
235 _run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir)
238 def _get_esptool_args(args):
239 esptool_path = os.path.join(os.environ["IDF_PATH"], "components/esptool_py/esptool/esptool.py")
240 if args.port is None:
241 args.port = get_default_serial_port()
242 result = [ PYTHON, esptool_path ]
243 result += [ "-p", args.port ]
244 result += [ "-b", str(args.baud) ]
247 def flash(action, args):
249 Run esptool to flash the entire project, from an argfile generated by the build system
251 flasher_args_path = { # action -> name of flasher args file generated by build system
252 "bootloader-flash": "flash_bootloader_args",
253 "partition_table-flash": "flash_partition_table_args",
254 "app-flash": "flash_app_args",
255 "flash": "flash_project_args",
257 esptool_args = _get_esptool_args(args)
258 esptool_args += [ "write_flash", "@"+flasher_args_path ]
259 _run_tool("esptool.py", esptool_args, args.build_dir)
262 def erase_flash(action, args):
263 esptool_args = _get_esptool_args(args)
264 esptool_args += [ "erase_flash" ]
265 _run_tool("esptool.py", esptool_args, args.build_dir)
268 def monitor(action, args):
270 Run idf_monitor.py to watch build output
272 if args.port is None:
273 args.port = get_default_serial_port()
274 desc_path = os.path.join(args.build_dir, "project_description.json")
275 if not os.path.exists(desc_path):
276 _ensure_build_directory(args)
277 with open(desc_path, "r") as f:
278 project_desc = json.load(f)
280 elf_file = os.path.join(args.build_dir, project_desc["app_elf"])
281 if not os.path.exists(elf_file):
282 raise FatalError("ELF file '%s' not found. You need to build & flash the project before running 'monitor', and the binary on the device must match the one in the build directory exactly. Try 'idf.py flash monitor'." % elf_file)
283 idf_monitor = os.path.join(os.environ["IDF_PATH"], "tools/idf_monitor.py")
284 monitor_args = [PYTHON, idf_monitor ]
285 if args.port is not None:
286 monitor_args += [ "-p", args.port ]
287 monitor_args += [ "-b", project_desc["monitor_baud"] ]
288 monitor_args += [ elf_file ]
290 idf_py = [ PYTHON ] + get_commandline_options() # commands to re-run idf.py
291 monitor_args += [ "-m", " ".join("'%s'" % a for a in idf_py) ]
293 if "MSYSTEM" is os.environ:
294 monitor_args = [ "winpty" ] + monitor_args
295 _run_tool("idf_monitor", monitor_args, args.project_dir)
298 def clean(action, args):
299 if not os.path.isdir(args.build_dir):
300 print("Build directory '%s' not found. Nothing to clean." % args.build_dir)
302 build_target("clean", args)
304 def reconfigure(action, args):
305 _ensure_build_directory(args, True)
307 def fullclean(action, args):
308 build_dir = args.build_dir
309 if not os.path.isdir(build_dir):
310 print("Build directory '%s' not found. Nothing to clean." % build_dir)
312 if len(os.listdir(build_dir)) == 0:
313 print("Build directory '%s' is empty. Nothing to clean." % build_dir)
316 if not os.path.exists(os.path.join(build_dir, "CMakeCache.txt")):
317 raise FatalError("Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically delete files in this directory. Delete the directory manually to 'clean' it." % build_dir)
318 red_flags = [ "CMakeLists.txt", ".git", ".svn" ]
319 for red in red_flags:
320 red = os.path.join(build_dir, red)
321 if os.path.exists(red):
322 raise FatalError("Refusing to automatically delete files in directory containing '%s'. Delete files manually if you're sure." % red)
323 # OK, delete everything in the build directory...
324 for f in os.listdir(build_dir): # TODO: once we are Python 3 only, this can be os.scandir()
325 f = os.path.join(build_dir, f)
331 def print_closing_message(args):
332 # print a closing message of some kind
335 if "flash" in str(args.actions):
339 # Otherwise, if we built any binaries print a message about
341 def print_flashing_message(title, key):
342 print("\n%s build complete. To flash, run this command:" % title)
344 with open(os.path.join(args.build_dir, "flasher_args.json")) as f:
345 flasher_args = json.load(f)
348 return os.path.relpath(os.path.join(args.build_dir, f))
350 if key != "project": # flashing a single item
352 if key == "bootloader": # bootloader needs --flash-mode, etc to be passed in
353 cmd = " ".join(flasher_args["write_flash_args"]) + " "
355 cmd += flasher_args[key]["offset"] + " "
356 cmd += flasher_path(flasher_args[key]["file"])
357 else: # flashing the whole project
358 cmd = " ".join(flasher_args["write_flash_args"]) + " "
359 flash_items = sorted(((o,f) for (o,f) in flasher_args["flash_files"].items() if len(o) > 0),
360 key = lambda (o,_): int(o, 0))
361 for o,f in flash_items:
362 cmd += o + " " + flasher_path(f) + " "
364 print("%s -p %s -b %s write_flash %s" % (
365 os.path.relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]),
366 args.port or "(PORT)",
369 print("or run 'idf.py -p %s %s'" % (args.port or "(PORT)", key + "-flash" if key != "project" else "flash",))
371 if "all" in args.actions or "build" in args.actions:
372 print_flashing_message("Project", "project")
374 if "app" in args.actions:
375 print_flashing_message("App", "app")
376 if "partition_table" in args.actions:
377 print_flashing_message("Partition Table", "partition_table")
378 if "bootloader" in args.actions:
379 print_flashing_message("Bootloader", "bootloader")
382 # action name : ( function (or alias), dependencies, order-only dependencies )
383 "all" : ( build_target, [], [ "reconfigure", "menuconfig", "clean", "fullclean" ] ),
384 "build": ( "all", [], [] ), # build is same as 'all' target
385 "clean": ( clean, [], [ "fullclean" ] ),
386 "fullclean": ( fullclean, [], [] ),
387 "reconfigure": ( reconfigure, [], [ "menuconfig" ] ),
388 "menuconfig": ( build_target, [], [] ),
389 "confserver": ( build_target, [], [] ),
390 "size": ( build_target, [ "app" ], [] ),
391 "size-components": ( build_target, [ "app" ], [] ),
392 "size-files": ( build_target, [ "app" ], [] ),
393 "bootloader": ( build_target, [], [] ),
394 "bootloader-clean": ( build_target, [], [] ),
395 "bootloader-flash": ( flash, [ "bootloader" ], [ "erase_flash"] ),
396 "app": ( build_target, [], [ "clean", "fullclean", "reconfigure" ] ),
397 "app-flash": ( flash, [ "app" ], [ "erase_flash"]),
398 "partition_table": ( build_target, [], [ "reconfigure" ] ),
399 "partition_table-flash": ( flash, [ "partition_table" ], [ "erase_flash" ]),
400 "flash": ( flash, [ "all" ], [ "erase_flash" ] ),
401 "erase_flash": ( erase_flash, [], []),
402 "monitor": ( monitor, [], [ "flash", "partition_table-flash", "bootloader-flash", "app-flash" ]),
406 def get_commandline_options():
407 """ Return all the command line options up to but not including the action """
410 if a in ACTIONS.keys():
416 def get_default_serial_port():
417 """ Return a default serial port. esptool can do this (smarter), but it can create
418 inconsistencies where esptool.py uses one port and idf_monitor uses another.
420 Same logic as esptool.py search order, reverse sort by name and choose the first port.
422 ports = list(reversed(sorted(
423 p.device for p in serial.tools.list_ports.comports() )))
425 print ("Choosing default port %s (use '-p PORT' option to set a specific serial port)" % ports[0])
428 raise RuntimeError("No serial ports found. Connect a device, or use '-p PORT' option to set a specific port.")
432 if sys.version_info[0] != 2 or sys.version_info[1] != 7:
433 raise FatalError("ESP-IDF currently only supports Python 2.7, and this is Python %d.%d.%d. Search for 'Setting the Python Interpreter' in the ESP-IDF docs for some tips to handle this." % sys.version_info[:3])
435 parser = argparse.ArgumentParser(description='ESP-IDF build management tool')
436 parser.add_argument('-p', '--port', help="Serial port",
437 default=os.environ.get('ESPPORT', None))
438 parser.add_argument('-b', '--baud', help="Baud rate",
439 default=os.environ.get('ESPBAUD', 460800))
440 parser.add_argument('-C', '--project-dir', help="Project directory", default=os.getcwd())
441 parser.add_argument('-B', '--build-dir', help="Build directory", default=None)
442 parser.add_argument('-G', '--generator', help="Cmake generator", choices=GENERATOR_CMDS.keys())
443 parser.add_argument('-n', '--no-warnings', help="Disable Cmake warnings", action="store_true")
444 parser.add_argument('-v', '--verbose', help="Verbose build output", action="store_true")
445 parser.add_argument('--no-ccache', help="Disable ccache. Otherwise, if ccache is available on the PATH then it will be used for faster builds.", action="store_true")
446 parser.add_argument('actions', help="Actions (build targets or other operations)", nargs='+',
447 choices=ACTIONS.keys())
449 args = parser.parse_args()
453 # Advanced parameter checks
454 if args.build_dir is not None and os.path.realpath(args.project_dir) == os.path.realpath(args.build_dir):
455 raise FatalError("Setting the build directory to the project directory is not supported. Suggest dropping --build-dir option, the default is a 'build' subdirectory inside the project directory.")
456 if args.build_dir is None:
457 args.build_dir = os.path.join(args.project_dir, "build")
458 args.build_dir = os.path.realpath(args.build_dir)
460 completed_actions = set()
461 def execute_action(action, remaining_actions):
462 ( function, dependencies, order_dependencies ) = ACTIONS[action]
463 # very simple dependency management, build a set of completed actions and make sure
464 # all dependencies are in it
465 for dep in dependencies:
466 if not dep in completed_actions:
467 execute_action(dep, remaining_actions)
468 for dep in order_dependencies:
469 if dep in remaining_actions and not dep in completed_actions:
470 execute_action(dep, remaining_actions)
472 if action in completed_actions:
473 pass # we've already done this, don't do it twice...
474 elif function in ACTIONS: # alias of another action
475 execute_action(function, remaining_actions)
477 function(action, args)
479 completed_actions.add(action)
481 actions = list(args.actions)
482 while len(actions) > 0:
483 execute_action(actions[0], actions[1:])
486 print_closing_message(args)
488 if __name__ == "__main__":
491 except FatalError as e: