diff --git a/doc/faq/a06-global-build-options.rst b/doc/faq/a06-global-build-options.rst new file mode 100644 index 0000000000..24a5a12c92 --- /dev/null +++ b/doc/faq/a06-global-build-options.rst @@ -0,0 +1,252 @@ +How to specify global build defines and options +=============================================== + +To create globally usable macro definitions for a Sketch, create a file +with a name based on your Sketch’s file name followed by ``.globals.h`` +in the Sketch folder. For example, if the main Sketch file is named +``LowWatermark.ino``, its global ``.h`` file would be +``LowWatermark.ino.globals.h``. This file will be implicitly included +with every module built for your Sketch. Do not directly include it in +any of your sketch files or in any other source files. There is no need +to create empty/dummy files, when not used. + +This global ``.h`` also supports embedding compiler command-line options +in a unique “C” block comment. Compiler options are placed in a “C” +block comment starting with ``/*@create-file:build.opt@``. This +signature line must be alone on a single line. The block comment ending +``*/`` should also be alone on a single line. In between, place your +compiler command-line options just as you would have for the GCC @file +command option. + +Actions taken in processing comment block to create ``build.opt`` \* for +each line, white space is trimmed \* blank lines are skipped \* lines +starting with ``*``, ``//``, or ``#`` are skipped \* the remaining +results are written to build tree\ ``/core/build.opt`` \* multiple +``/*@create-file:build.opt@`` ``*/`` comment blocks are not allowed \* +``build.opt`` is finished with a ``-include ...`` command, which +references the global .h its contents were extracted from. + +Example Sketch: ``LowWatermark.ino`` + +.. code:: cpp + + #include // has prototype for umm_free_heap_size_min() + + void setup() { + Serial.begin(115200); + delay(200); + #ifdef MYTITLE1 + Serial.printf("\r\n" MYTITLE1 MYTITLE2 "\r\n"); + #else + Serial.println("ERROR: MYTITLE1 not present"); + #endif + Serial.printf("Heap Low Watermark %u\r\n", umm_free_heap_size_min()); + } + + void loop() {} + +Global ``.h`` file: ``LowWatermark.ino.globals.h`` + +.. code:: cpp + + /*@create-file:build.opt@ + // An embedded build.opt file using a "C" block comment. The starting signature + // must be on a line by itself. The closing block comment pattern should be on a + // line by itself. Each line within the block comment will be space trimmed and + // written to build.opt, skipping blank lines and lines starting with '//', '*' + // or '#'. + + * this line is ignored + # this line is ignored + -DMYTITLE1="\"Running on \"" + -O3 + //-fanalyzer + -DUMM_STATS_FULL=1 + */ + + #ifndef LOWWATERMARK_INO_GLOBALS_H + #define LOWWATERMARK_INO_GLOBALS_H + + #if !defined(__ASSEMBLER__) + // Defines kept away from assembler modules + // i.e. Defines for .cpp, .ino, .c ... modules + #endif + + #if defined(__cplusplus) + // Defines kept private to .cpp and .ino modules + //#pragma message("__cplusplus has been seen") + #define MYTITLE2 "Empty" + #endif + + #if !defined(__cplusplus) && !defined(__ASSEMBLER__) + // Defines kept private to .c modules + #define MYTITLE2 "Full" + #endif + + #if defined(__ASSEMBLER__) + // Defines kept private to assembler modules + #endif + + #endif + +Aggressively cache compiled core +================================ + +This feature appeared with the release of Arduino IDE 1.8.2. The feature +“Aggressively Cache Compiled core” refers to sharing a single copy of +``core.a`` across all Arduino IDE Sketch windows. This feature is on by +default. ``core.a`` is an archive file containing the compiled objects +of ``./core/esp8266/*``. Created after your 1ST successful compilation. +All other open sketch builds use this shared file. When you close all +Arduino IDE windows, the core archive file is deleted. + +This feature is not compatible with using global defines or compiler +command-line options. Without mediation, bad builds could result, when +left enabled. When ``#define`` changes require rebuilding ``core.a`` and +multiple Sketches are open, they can no longer reliably share one cached +``core.a``. In a simple case: The 1st Sketch to be built has its version +of ``core.a`` cached. Other sketches will use this cached version for +their builds. + +There are two solutions to this issue: 1. Turn off the “Aggressively +Cache Compiled core” feature, by setting ``compiler.cache_core=false``. +2. Rely on the not ideal fail-safe, aggressive cache workaround built +into the script. + +Using “compiler.cache_core=false” +--------------------------------- + +There are two ways to turn off the “Aggressively Cache Compiled core” +feature: This can be done with the Arduino IDE command-line or a text +editor. + +Using the Arduino IDE command-line from a system command line, enter the +following: + +:: + + arduino --pref compiler.cache_core=false --save-prefs + +For the text editor, you need to find the location of +``preferences.txt``. From the Arduino IDE, go to *File->Preferences*. +Make note of the path to ``prefereces.txt``. You *cannot* edit the file +while the Arduino IDE is running. Close all Arduino IDE windows and edit +the file ``preferences.txt``. Change ``compiler.cache_core=true`` to +``compiler.cache_core=false`` and save. Then each sketch will maintain +its *own* copy of ``core.a`` built with the customization expressed by +their respective ``build.opt`` file. + +The “workaround” +---------------- + +When the “Aggressively Cache Compiled core” feature is enabled and the +global define file is detected, a workaround will turn on and stay on. +When you switch between Sketch windows, core will be recompiled and the +cache updated. The workaround logic is reset when Arduino IDE is +completely shutdown and restarted. + +The workaround is not perfect. These issues may be of concern: 1. Dirty +temp space. Arduino build cache files left over from a previous run or +boot. 2. Arduino command-line options: \* override default +preferences.txt file. \* override a preference, specifically +``compiler.cache_core``. 3. Multiple versions of the Arduino IDE running + +**Dirty temp space** + +A minor concern, the workaround is always on. Not an issue for build +accuracy, but ``core.a`` maybe rebuild more often than necessary. + +Some operating systems are better at cleaning up their temp space than +others at reboot after a crash. At least for Windows®, you may need to +manually delete the Arduino temp files and directories after a crash. +Otherwise, the workaround logic may be left on. There is no harm in the +workaround being stuck on, the build will be correct; however, the core +files will occasionally be recompiled when not needed. + +For some Windows® systems the temp directory can be found near +``C:\Users\\AppData\Local\Temp\arduino*``. Note ``AppData`` is +a hidden directory. For help with this do an Internet search on +``windows disk cleanup``. Or, type ``disk cleanup`` in the Windows® +taskbar search box. + +With Linux, this problem could occur after an Arduino IDE crash. The +problem would be cleared after a reboot. Or you can manually cleanup the +``/tmp/`` directory before restarting the Arduino IDE. + +**Arduino command-line option overrides** + +The script needs to know the working value of ``compiler.cache_core`` +that the Arduino IDE uses when building. This script can learn the state +through documented locations; however, the Arduino IDE has two +command-line options that can alter the results the Arduino IDE uses +internally. And, the Arduino IDE does not provide a means for a script +to learn the override value. + +These two command-line options are the problem: + +:: + + ./arduino --preferences-file other-preferences.txt + ./arduino --pref compiler.cache_core=false + +Hints for discovering the value of ``compiler.cache_core``, can be +provided by specifying ``mkbuildoptglobals.extra_flags=...`` in +``platform.local.txt``. + +Examples of hints: + +:: + + mkbuildoptglobals.extra_flags=--preferences_sketch # assume file preferences.txt in the sketch folder + mkbuildoptglobals.extra_flags=--preferences_sketch "pref.txt" # is relative to the sketch folder + mkbuildoptglobals.extra_flags=--no_cache_core + mkbuildoptglobals.extra_flags=--cache_core + mkbuildoptglobals.extra_flags=--preferences_file "other-preferences.txt" # relative to IDE or full path + +If required, remember to quote file or file paths. + +**Multiple versions of the Arduino IDE running** + +You can run multiple Arduino IDE windows as long as you run one version +of the Arduino IDE at a time. When testing different versions, +completely exit one before starting the next version. For example, +Arduino IDE 1.8.19 and Arduino IDE 2.0 work with different temp and +build paths. With this combination, the workaround logic sometimes fails +to enable. + +At the time of this writing, when Arduino IDE 2.0 rc5 exits, it leaves +the temp space dirty. This keeps the workaround active the next time the +IDE is started. If this is an issue, manually delete the temp files. + +Custom build environments +========================= + +Some custom build environments may have already addressed this issue by +other means. If you have a custom build environment that does not +require this feature and would like to turn it off, you can add the +following lines to the ``platform.local.txt`` used in your build +environment: + +:: + + recipe.hooks.prebuild.2.pattern= + build.opt.flags= + +Other build confusion +===================== + +1. Renaming a file does not change the last modified timestamp, possibly + causing issues when adding a file by renaming and rebuilding. A good + example of this problem would be to have then fixed a typo in file + name ``LowWatermark.ino.globals.h``. You need to touch (update + timestamp) the file so a “rebuild all” is performed. + +2. When a ``.h`` file is renamed in the sketch folder, a copy of the old + file remains in the build sketch folder. This can create confusion if + you missed an edit in updating an ``#include`` in one or more of your + modules. That module will continue to use the stale version of the + ``.h`` until you restart the IDE or other major changes that would + cause the IDE to delete and recopy the contents from the source + Sketch directory. Changes on the IDE Tools board settings may cause a + complete rebuild, clearing the problem. This may be the culprit for + “What! It built fine last night!” diff --git a/doc/faq/readme.rst b/doc/faq/readme.rst index 53e830358b..04ccd74880 100644 --- a/doc/faq/readme.rst +++ b/doc/faq/readme.rst @@ -191,3 +191,12 @@ How to resolve "undefined reference to ``flashinit`'" error ? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Please read `flash layout <../filesystem.rst>`__ documentation entry. + +How to specify global build defines and options? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By using a uniquely named `.h` file, macro definitions can be created and +globally used. Additionally, compiler command-line options can be embedded in +this file as a unique block comment. + +`Read more `__. diff --git a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino new file mode 100644 index 0000000000..2bc18751e1 --- /dev/null +++ b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino @@ -0,0 +1,31 @@ +/* + * Showcase the use of embedded build options and global defines through a specially named .h file. + * Sketch file name followed by ".globals.h", "GlobalBuildOptions.ino.globals.h" + * + * Example from https://arduino-esp8266.readthedocs.io/en/latest/faq/a06-global-build-options.html + * + * Note, we do not "#include" the special file "GlobalBuildOptions.ino.globals.h". + * The prebuild script will make it available to all modules. + * + * To track the new sketch name when saving this sketch to a new location and + * name, remember to update the global .h file name. + */ + +#include // has prototype for umm_free_heap_size_min() + +void setup() { + Serial.begin(115200); + delay(200); + +#ifdef MYTITLE1 + Serial.printf("\r\n" MYTITLE1 MYTITLE2 "\r\n"); +#else + Serial.println("ERROR: MYTITLE1 not present"); +#endif + +#if defined(UMM_STATS_FULL) + Serial.printf("Heap Low Watermark %u\r\n", umm_free_heap_size_min()); +#endif +} + +void loop() {} diff --git a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h new file mode 100644 index 0000000000..12a4882ff8 --- /dev/null +++ b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h @@ -0,0 +1,39 @@ +/*@create-file:build.opt@ + // An embedded build.opt file using a "C" block comment. The starting signature + // must be on a line by itself. The closing block comment pattern should be on a + // line by itself. Each line within the block comment will be space trimmed and + // written to build.opt, skipping blank lines and lines starting with '//', '*' + // or '#'. + -DMYTITLE1="\"Running on \"" + * this line is ignored + *@create-file:build.opt@ + # this line is ignored + -O3 + // -fanalyzer + -DUMM_STATS_FULL=1 +*/ + +#ifndef GLOBALBUILDOPTIONS_INO_GLOBALS_H +#define GLOBALBUILDOPTIONS_INO_GLOBALS_H + +#if !defined(__ASSEMBLER__) +// Defines kept away from assembler modules +// i.e. Defines for .cpp, .ino, .c ... modules +#endif + +#if defined(__cplusplus) +// Defines kept private to .cpp and .ino modules +//#pragma message("__cplusplus has been seen") +#define MYTITLE2 "Empty" +#endif + +#if !defined(__cplusplus) && !defined(__ASSEMBLER__) +// Defines kept private to .c modules +#define MYTITLE2 "~Full" +#endif + +#if defined(__ASSEMBLER__) +// Defines kept private to assembler modules +#endif + +#endif diff --git a/platform.txt b/platform.txt index 2f0cfded80..ffdd64f321 100644 --- a/platform.txt +++ b/platform.txt @@ -17,6 +17,7 @@ runtime.tools.signing={runtime.platform.path}/tools/signing.py runtime.tools.elf2bin={runtime.platform.path}/tools/elf2bin.py runtime.tools.sizes={runtime.platform.path}/tools/sizes.py runtime.tools.makecorever={runtime.platform.path}/tools/makecorever.py +runtime.tools.mkbuildoptglobals={runtime.platform.path}/tools/mkbuildoptglobals.py runtime.tools.mkdir={runtime.platform.path}/tools/mkdir.py runtime.tools.cp={runtime.platform.path}/tools/cp.py runtime.tools.eboot={runtime.platform.path}/bootloaders/eboot/eboot.elf @@ -58,11 +59,18 @@ build.spiffs_start= build.spiffs_end= build.spiffs_blocksize= +# Fully qualified file names for processing sketch global options +globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h +commonhfile.fqfn={build.core.path}/CommonHFile.h +build.opt.fqfn={build.path}/core/build.opt +build.opt.flags="@{build.opt.fqfn}" +mkbuildoptglobals.extra_flags= + compiler.path={runtime.tools.xtensa-lx106-elf-gcc.path}/bin/ compiler.sdk.path={runtime.platform.path}/tools/sdk compiler.libc.path={runtime.platform.path}/tools/sdk/libc/xtensa-lx106-elf -compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" +compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 {build.opt.flags} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" # support precompiled libraries in IDE v1.8.6+ compiler.libraries.ldflags= @@ -107,7 +115,11 @@ compiler.elf2hex.extra_flags= ## needs git recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h" # This is quite a working hack. This form of prebuild hook, while intuitive, is not explicitly documented. -recipe.hooks.prebuild.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --build_path "{build.path}" --platform_path "{runtime.platform.path}" --version "{version}" +recipe.hooks.prebuild.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --build_path "{build.path}" --platform_path "{runtime.platform.path}" --version "{version}" + +# Handle processing sketch global options +recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} + ## Build the app.ld linker file recipe.hooks.linking.prelink.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkdir}" -p "{build.path}/ld_h/" diff --git a/tests/build.sh b/tests/build.sh index 45f88e7ef2..5f9e1be4cb 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -19,4 +19,3 @@ install_arduino nodebug build_sketches_with_arduino "$mod" "$rem" lm2f rm -rf "$cache_dir" - diff --git a/tests/common.sh b/tests/common.sh index fc62aa7e2e..9628c34cba 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -67,6 +67,8 @@ function build_sketches() local sketches=$(find $srcpath -name *.ino | sort) print_size_info >size.log export ARDUINO_IDE_PATH=$arduino + local k_partial_core_cleanup=("build.opt" "*.ino.globals.h") + local mk_clean_core=1 local testcnt=0 for sketch in $sketches; do testcnt=$(( ($testcnt + 1) % $build_mod )) @@ -74,6 +76,26 @@ function build_sketches() continue # Not ours to do fi + # mkbuildoptglobals.py is optimized around the Arduino IDE 1.x + # behaviour. One way the CI differs from the Arduino IDE is in the + # handling of core and caching core. With the Arduino IDE, each sketch + # has a private copy of core and contributes to a core cache. With the + # CI, there is one shared copy of core for all sketches. When global + # options are used, the shared copy of core and cache are removed before + # and after the build. + # + # Do we need a clean core build? $build_dir/core/* cannot be shared + # between sketches when global options are present. + if [ -s ${sketch}.globals.h ]; then + mk_clean_core=1 + fi + if [ $mk_clean_core -ne 0 ]; then + rm -rf rm $build_dir/core/* + else + # Remove sketch specific files from ./core/ between builds. + rm -rf $build_dir/core/build.opt $build_dir/core/*.ino.globals.h + fi + if [ -e $cache_dir/core/*.a ]; then # We need to preserve the build.options.json file and replace the last .ino # with this sketch's ino file, or builder will throw everything away. @@ -82,6 +104,17 @@ function build_sketches() # Set the time of the cached core.a file to the future so the GIT header # we regen won't cause the builder to throw it out and rebuild from scratch. touch -d 'now + 1 day' $cache_dir/core/*.a + if [ $mk_clean_core -ne 0 ]; then + # Hack workaround for CI not handling core rebuild for global options + rm $cache_dir/core/*.a + fi + fi + + if [ -s ${sketch}.globals.h ]; then + # Set to cleanup core at the start of the next build. + mk_clean_core=1 + else + mk_clean_core=0 fi # Clear out the last built sketch, map, elf, bin files, but leave the compiled @@ -197,6 +230,7 @@ function install_ide() # Set custom warnings for all builds (i.e. could add -Wextra at some point) echo "compiler.c.extra_flags=-Wall -Wextra -Werror $debug_flags" > esp8266/platform.local.txt echo "compiler.cpp.extra_flags=-Wall -Wextra -Werror $debug_flags" >> esp8266/platform.local.txt + echo "mkbuildoptglobals.extra_flags=--ci --cache_core" >> esp8266/platform.local.txt echo -e "\n----platform.local.txt----" cat esp8266/platform.local.txt echo -e "\n----\n" @@ -250,4 +284,3 @@ if [ -z "$TRAVIS_BUILD_DIR" ]; then popd > /dev/null echo "TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR" fi - diff --git a/tools/build.py b/tools/build.py index 791ac30e2a..378c4a9f9d 100755 --- a/tools/build.py +++ b/tools/build.py @@ -73,7 +73,7 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args): fqbn += ',waveform=phase' cmd += [fqbn] cmd += ['-built-in-libraries', ide_path + '/libraries'] - cmd += ['-ide-version=10607'] + cmd += ['-ide-version=10802'] cmd += ['-warnings={warnings}'.format(**vars(args))] if args.verbose: cmd += ['-verbose'] diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py new file mode 100644 index 0000000000..188b07b42a --- /dev/null +++ b/tools/mkbuildoptglobals.py @@ -0,0 +1,838 @@ +#!/usr/bin/env python3 + +# This script manages the use of a file with a unique name, like +# `Sketch.ino.globals.h`, in the Sketch source directory to provide compiler +# command-line options (build options) and sketch global macros. The build +# option data is encapsulated in a unique "C" comment block and extracted into +# the build tree during prebuild. +# +# Copyright (C) 2022 - M Hightower +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A Tip of the hat to: +# +# This PR continues the effort to get some form of global build support +# presented by brainelectronics' PR https://github.com/esp8266/Arduino/pull/8095 +# +# Used d-a-v's global name suggestion from arduino PR +# https://github.com/arduino/arduino-cli/pull/1524 +# +""" +Operation + +"Sketch.ino.globals.h" - A global h file in the Source Sketch directory. The +string Sketch.ino is the actual name of the sketch program. A matching copy is +kept in the build path/core directory. The file is empty when it does not exist +in the source directory. + +Using Sketch.ino.globals.h as a container to hold build.opt, gives implicit +dependency tracking for build.opt by way of Sketch.ino.globals.h's +dependencies. +Example: + gcc ... @{build.path}/core/build.opt -include "{build.path}/core/{build.project_name}.globals.h" ... + +In this implementation the '-include "{build.path}/core/{build.project_name}.globals.h"' +component is added to the build.opt file. + gcc ... @{build.path}/core/build.opt ... + +At each build cycle, "{build.project_name}.globals.h" is conditoinally copied to +"{build.path}/core/" at prebuild, and build.opt is extraction as needed. The +Sketch.ino.globals.h's dependencies will trigger "rebuild all" as needed. + +If Sketch.ino.globals.h is not in the source sketch folder, an empty +versions is created in the build tree. The file build.opt always contains a +"-include ..." entry so that file dependencies are generated for +Sketch.ino.globals.h. This allows for change detection when the file is +added. +""" + +""" +Arduino `preferences.txt` changes + +"Aggressively cache compiled core" ideally should be turned off; however, +a workaround has been implimented. +In ~/.arduino15/preferences.txt, to disable the feature: + compiler.cache_core=false + +Reference: +https://forum.arduino.cc/t/no-aggressively-cache-compiled-core-in-ide-1-8-15/878954/2 +""" + +""" +# Updates or Additions for platform.txt or platform.local.txt + +runtime.tools.mkbuildoptglobals={runtime.platform.path}/tools/mkbuildoptglobals.py + +# Fully qualified file names for processing sketch global options +globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h +commonhfile.fqfn={build.core.path}/CommonHFile.h +build.opt.fqfn={build.path}/core/build.opt +mkbuildoptglobals.extra_flags= + +recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} + +compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 @{build.opt.path} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" +""" + +""" +A Sketch.ino.globals.h file with embedded build.opt might look like this + +/*@create-file:build.opt@ +// An embedded build.opt file using a "C" block comment. The starting signature +// must be on a line by itself. The closing block comment pattern should be on a +// line by itself. Each line within the block comment will be space trimmed and +// written to build.opt, skipping blank lines and lines starting with '//', '*' +// or '#'. + +-DMYDEFINE="\"Chimichangas do not exist\"" +-O3 +-fanalyzer +-DUMM_STATS=2 +*/ + +#ifndef SKETCH_INO_GLOBALS_H +#define SKETCH_INO_GLOBALS_H + +#if defined(__cplusplus) +// Defines kept private to .cpp modules +//#pragma message("__cplusplus has been seen") +#endif + +#if !defined(__cplusplus) && !defined(__ASSEMBLER__) +// Defines kept private to .c modules +#endif + +#if defined(__ASSEMBLER__) +// Defines kept private to assembler modules +#endif + +#endif +""" + +""" +Added 2) and 5) to docs + +Caveats, Observations, and Ramblings + +1) Edits to platform.txt or platform.local.txt force a complete rebuild that +removes the core folder. Not a problem, just something to be aware of when +debugging this script. Similarly, changes on the IDE Tools selection cause a +complete rebuild. + +In contrast, the core directory is not deleted when the rebuild occurs from +changing a file with an established dependency. + +2) Renaming files does not change the last modified timestamp, possibly causing +issues when replacing files by renaming and rebuilding. + +A good example of this problem is when you correct the spelling of file +Sketch.ino.globals.h. You need to touch (update time stampt) the file so a +rebuild all is performed. + +3) During the build two identical copies of Sketch.ino.globals.h will exist. +#ifndef fencing will be needed for non comment blocks in Sketch.ino.globals.h. + +4) By using a .h file to encapsulate "build.opt" options, the information is not +lost after a save-as. Before with an individual "build.opt" file, the file was +missing in the saved copy. + +5) When a .h file is renamed, a copy of the old file remains in the build +sketch folder. This can create confusion if you missed an edit in updating an +include in one or more of your modules. That module will continue to use the +stale version of the .h, until you restart the IDE or other major changes that +would cause the IDE to delete and recopy the contents from the source sketch. + +This may be the culprit for "What! It built fine last night!" + +6a) In The case of two Arduino IDE screens up with different programs, they can +share the same core archive file. Defines on one screen will change the core +archive, and a build on the 2nd screen will build with those changes. +The 2nd build will have the core built for the 1st screen. It gets uglier. With +the 2nd program, the newly built modules used headers processed with different +defines than the core. + +6b) Problem: Once core has been build, changes to build.opt or globals.h will +not cause the core archive to be rebuild. You either have to change tool +settings or close and reopen the Arduino IDE. This is a variation on 6a) above. +I thought this was working for the single sketch case, but it does not! :( +That is because sometimes it does build properly. What is unknown are the +causes that will make it work and fail? + * Fresh single Arduino IDE Window, open with file to build - works + +I think these, 6a and 6b, are resolved by setting `compiler.cache_core=false` +in ~/.arduino15/preferences.txt, to disable the aggressive caching feature: + https://forum.arduino.cc/t/no-aggressively-cache-compiled-core-in-ide-1-8-15/878954/2 + +Added workaround for `compiler.cache_core=true` case. +See `if use_aggressive_caching_workaround:` in main(). + +7) Suspected but not confirmed. A quick edit and rebuild don't always work well. +Build does not work as expected. This does not fail often. Maybe PIC NIC. +""" + +import argparse +from shutil import copyfile +import glob +import os +import platform +import sys +import textwrap +import time + +# Need to work on signature line used for match to avoid conflicts with +# existing embedded documentation methods. +build_opt_signature = "/*@create-file:build.opt@" + +docs_url = "https://arduino-esp8266.readthedocs.io/en/latest/faq/a06-global-build-options.html" + + +err_print_flag = False +msg_print_buf = "" +debug_enabled = False + +# Issues trying to address through buffered printing +# 1. Arduino IDE 2.0 RC5 does not show stderr text in color. Text printed does +# not stand out from stdout messages. +# 2. Separate pipes, buffering, and multiple threads with output can create +# mixed-up messages. "flush" helped but did not resolve. The Arduino IDE 2.0 +# somehow makes the problem worse. +# 3. With Arduino IDE preferences set for "no verbose output", you only see +# stderr messages. Prior related prints are missing. +# +# Locally buffer and merge both stdout and stderr prints. This allows us to +# print a complete context when there is an error. When any buffered prints +# are targeted to stderr, print the whole buffer to stderr. + +def print_msg(*args, **kwargs): + global msg_print_buf + if 'sep' in kwargs: + sep = kwargs['sep'] + else: + sep = ' ' + + msg_print_buf += args[0] + for arg in args[1:]: + msg_print_buf += sep + msg_print_buf += arg + + if 'end' in kwargs: + msg_print_buf += kwargs['end'] + else: + msg_print_buf += '\n' + + +# Bring attention to errors with a blank line and lines starting with "*** ". +def print_err(*args, **kwargs): + global err_print_flag + if (args[0])[0] != ' ': + print_msg("") + print_msg("***", *args, **kwargs) + err_print_flag = True + +def print_dbg(*args, **kwargs): + global debug_enabled + global err_print_flag + if debug_enabled: + print_msg("DEBUG:", *args, **kwargs) + err_print_flag = True + + +def handle_error(err_no): + # on err_no 0, commit print buffer to stderr or stdout + # on err_no != 0, commit print buffer to stderr and sys exist with err_no + global msg_print_buf + global err_print_flag + if len(msg_print_buf): + if err_no or err_print_flag: + fd = sys.stderr + else: + fd = sys.stdout + print(msg_print_buf, file=fd, end='', flush=True) + msg_print_buf = "" + err_print_flag = False + if err_no: + sys.exit(err_no) + + +def copy_create_build_file(source_fqfn, build_target_fqfn): + """ + Conditionally copy a newer file between the source directory and the build + directory. When source file is missing, create an empty file in the build + directory. + return True when file change detected. + """ + if os.path.exists(source_fqfn): + if os.path.exists(build_target_fqfn) and \ + os.path.getmtime(build_target_fqfn) >= os.path.getmtime(source_fqfn): + # only copy newer files - do nothing, all is good + print_dbg(f"up to date os.path.exists({source_fqfn}) ") + return False + else: + # The new copy gets stamped with the current time, just as other + # files copied by `arduino-builder`. + copyfile(source_fqfn, build_target_fqfn) + print_dbg(f"copyfile({source_fqfn}, {build_target_fqfn})") + else: + if os.path.exists(build_target_fqfn) and \ + os.path.getsize(build_target_fqfn) == 0: + return False + else: + # Place holder - Must have an empty file to satisfy parameter list + # specifications in platform.txt. + with open(build_target_fqfn, 'w'): + pass + return True # file changed + + +def add_include_line(build_opt_fqfn, include_fqfn): + if not os.path.exists(include_fqfn): + # If file is missing, we need an place holder + with open(include_fqfn, 'w'): + pass + print("add_include_line: Created " + include_fqfn) + with open(build_opt_fqfn, 'a') as build_opt: + build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n') + + +def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn): + """ + Extract the embedded build.opt from Sketch.ino.globals.h into build + path/core/build.opt. The subdirectory path must already exist as well as the + copy of Sketch.ino.globals.h. + """ + global build_opt_signature + + build_opt = open(build_opt_fqfn, 'w') + if not os.path.exists(globals_h_fqfn) or (0 == os.path.getsize(globals_h_fqfn)): + build_opt.close() + return False + + complete_comment = False + build_opt_error = False + line_no = 0 + # If the source sketch did not have the file Sketch.ino.globals.h, an empty + # file was created in the ./core/ folder. + # By using the copy, open will always succeed. + with open(globals_h_fqfn, 'r') as src: + for line in src: + line = line.strip() + line_no += 1 + if line == build_opt_signature: + if complete_comment: + build_opt_error = True + print_err(" Multiple embedded build.opt blocks in", f'{file_name}:{line_no}') + continue + print_msg("Extracting embedded compiler command-line options from", f'{file_name}:{line_no}') + for line in src: + line = line.strip() + line_no += 1 + if 0 == len(line): + continue + if line.startswith("*/"): + complete_comment = True + break + elif line.startswith("*"): # these are so common - skip these should they occur + continue + elif line.startswith("#"): # allow some embedded comments + continue + elif line.startswith("//"): + continue + # some consistency checking before writing - give some hints about what is wrong + elif line == build_opt_signature: + print_err(" Double begin before end for embedded build.opt block in", f'{file_name}:{line_no}') + build_opt_error = True + elif line.startswith(build_opt_signature): + print_err(" build.opt signature block ignored, trailing character for embedded build.opt block in", f'{file_name}:{line_no}') + build_opt_error = True + elif "/*" in line or "*/" in line : + print_err(" Nesting issue for embedded build.opt block in", f'{file_name}:{line_no}') + build_opt_error = True + else: + print_msg(" ", f'{line_no:2}, Add command-line option: {line}', sep='') + build_opt.write(line + "\n") + elif line.startswith(build_opt_signature): + print_err(" build.opt signature block ignored, trailing character for embedded build.opt block in", f'{file_name}:{line_no}') + build_opt_error = True + if not complete_comment or build_opt_error: + build_opt.truncate(0) + build_opt.close() + if build_opt_error: + # this will help the script start over when the issue is fixed + os.remove(globals_h_fqfn) + print_err(" Extraction failed") + # Don't let the failure get hidden by a spew of nonsensical error + # messages that will follow. Bring things to a halt. + handle_error(1) + return False # not reached + elif complete_comment: + print_msg(" Created compiler command-line options file " + build_opt_fqfn) + build_opt.close() + return complete_comment + + +def enable_override(enable, commonhfile_fqfn): + # Reduce disk IO writes + if os.path.exists(commonhfile_fqfn): + if os.path.getsize(commonhfile_fqfn): # workaround active + if enable: + return + elif not enable: + return + with open(commonhfile_fqfn, 'w') as file: + if enable: + file.write("//Override aggressive caching\n") + # enable workaround when getsize(commonhfile_fqfn) is non-zero, disabled when zero + + +def discover_1st_time_run(build_path): + # Need to know if this is the 1ST compile of the Arduino IDE starting. + # Use empty cache directory as an indicator for 1ST compile. + # Arduino IDE 2.0 RC5 does not cleanup on exist like 1.6.19. Probably for + # debugging like the irregular version number 10607. For RC5 this indicator + # will be true after a reboot instead of a 1ST compile of the IDE starting. + # Another issue for this technique, Windows does not clear the Temp directory. :( + tmp_path, build = os.path.split(build_path) + ide_2_0 = 'arduino-sketch-' + if ide_2_0 == build[:len(ide_2_0)]: + search_path = os.path.join(tmp_path, 'arduino-core-cache/*') # Arduino IDE 2.0 + else: + search_path = os.path.join(tmp_path, 'arduino_cache_*/*') # Arduino IDE 1.6.x and up + + count = 0 + for dirname in glob.glob(search_path): + count += 1 + return 0 == count + + +def find_preferences_txt(runtime_ide_path): + """ + Check for perferences.txt in well-known locations. Most OSs have two + possibilities. When "portable" is present, it takes priority. Otherwise, the + remaining path wins. However, Windows has two. Depending on the install + source, the APP store or website download, both may appear and create an + ambiguous result. + + Return two item list - Two non "None" items indicate an ambiguous state. + + OS Path list for Arduino IDE 1.6.0 and newer + from: https://www.arduino.cc/en/hacking/preferences + """ + platform_name = platform.system() + if "Linux" == platform_name: + # Test for portable 1ST + # /portable/preferences.txt (when used in portable mode) + # For more on portable mode see https://docs.arduino.cc/software/ide-v1/tutorials/PortableIDE + fqfn = os.path.normpath(runtime_ide_path + "/portable/preferences.txt") + # Linux - verified with Arduino IDE 1.8.19 + if os.path.exists(fqfn): + return [fqfn, None] + fqfn = os.path.expanduser("~/.arduino15/preferences.txt") + # Linux - verified with Arduino IDE 1.8.18 and 2.0 RC5 64bit and AppImage + if os.path.exists(fqfn): + return [fqfn, None] + elif "Windows" == platform_name: + fqfn = os.path.normpath(runtime_ide_path + "\portable\preferences.txt") + # verified on Windows 10 with Arduino IDE 1.8.19 + if os.path.exists(fqfn): + return [fqfn, None] + # It is never simple. Arduino from the Windows APP store or the download + # Windows 8 and up option will save "preferences.txt" in one location. + # The downloaded Windows 7 (and up version) will put "preferences.txt" + # in a different location. When both are present due to various possible + # scenarios, use the more modern. + fqfn = os.path.expanduser("~\Documents\ArduinoData\preferences.txt") + # Path for "Windows app" - verified on Windows 10 with Arduino IDE 1.8.19 from APP store + fqfn2 = os.path.expanduser("~\AppData\local\Arduino15\preferences.txt") + # Path for Windows 7 and up - verified on Windows 10 with Arduino IDE 1.8.19 + if os.path.exists(fqfn): + if os.path.exists(fqfn2): + print_err("Multiple 'preferences.txt' files found:") + print_err(" " + fqfn) + print_err(" " + fqfn2) + return [fqfn, None] + else: + return [fqfn, fqfn2] + elif os.path.exists(fqfn2): + return [fqfn2, None] + elif "Darwin" == platform_name: + # Portable is not compatable with Mac OS X + # see https://docs.arduino.cc/software/ide-v1/tutorials/PortableIDE + fqfn = os.path.expanduser("~/Library/Arduino15/preferences.txt") + # Mac OS X - unverified + if os.path.exists(fqfn): + return [fqfn, None] + + print_err("File preferences.txt not found on " + platform_name) + return [None, None] + + +def get_preferences_txt(file_fqfn, key): + # Get Key Value, key is allowed to be missing. + # We assume file file_fqfn exists + basename = os.path.basename(file_fqfn) + with open(file_fqfn) as file: + for line in file: + name, value = line.partition("=")[::2] + if name.strip().lower() == key: + val = value.strip().lower() + if val != 'true': + val = False + print_msg(f" {basename}: {key}={val}") + return val + print_err(f" Key '{key}' not found in file {basename}. Default to true.") + return True # If we don't find it just assume it is set True + + +def check_preferences_txt(runtime_ide_path, preferences_file): + key = "compiler.cache_core" + # return the state of "compiler.cache_core" found in preferences.txt + if preferences_file != None: + if os.path.exists(preferences_file): + print_msg(f"Using preferences from '{preferences_file}'") + return get_preferences_txt(preferences_file, key) + else: + print_err(f"Override preferences file '{preferences_file}' not found.") + + elif runtime_ide_path != None: + # For a particular install, search the expected locations for platform.txt + # This should never fail. + file_fqfn = find_preferences_txt(runtime_ide_path) + if file_fqfn[0] != None: + print_msg(f"Using preferences from '{file_fqfn[0]}'") + val0 = get_preferences_txt(file_fqfn[0], key) + val1 = val0 + if file_fqfn[1] != None: + val1 = get_preferences_txt(file_fqfn[1], key) + if val0 == val1: # We can safely ignore that there were two preferences.txt files + return val0 + else: + print_err(f"Found too many preferences.txt files with different values for '{key}'") + raise UserWarning + else: + # Something is wrong with the installation or our understanding of the installation. + print_err("'preferences.txt' file missing from well known locations.") + + return None + + +def touch(fname, times=None): + with open(fname, "a") as file: + os.utime(file.fileno(), times) + +def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): + global debug_enabled + # touch both files with the same timestamp + touch(globals_h_fqfn) + with open(globals_h_fqfn, 'r') as file: + ts = os.stat(file.fileno()) + with open(commonhfile_fqfn, 'a') as file2: + os.utime(file2.fileno(), ns=(ts.st_atime_ns, ts.st_mtime_ns)) + + if debug_enabled: + print_dbg("After synchronous_touch") + ts = os.stat(globals_h_fqfn) + print_dbg(f" globals_h_fqfn ns_stamp = {ts.st_mtime_ns}") + print_dbg(f" getmtime(globals_h_fqfn) {os.path.getmtime(globals_h_fqfn)}") + ts = os.stat(commonhfile_fqfn) + print_dbg(f" commonhfile_fqfn ns_stamp = {ts.st_mtime_ns}") + print_dbg(f" getmtime(commonhfile_fqfn) {os.path.getmtime(commonhfile_fqfn)}") + +def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): + global docs_url + print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") + if args.runtime_ide_version < 10802: # CI also has version 10607 -- and args.runtime_ide_version != 10607: + # Aggresive core caching - not implemented before version 1.8.2 + # Note, Arduino IDE 2.0 rc5 has version 1.6.7 and has aggressive caching. + print_dbg(f"Old version ({args.runtime_ide_version}) of Arduino IDE no aggressive caching option") + return False + elif args.cache_core != None: + print_msg(f"Preferences override, this prebuild script assumes the 'compiler.cache_core' parameter is set to {args.cache_core}") + print_msg(f"To change, modify 'mkbuildoptglobals.extra_flags=(--cache_core | --no_cache_core)' in 'platform.local.txt'") + return args.cache_core + else: + ide_path = None + preferences_fqfn = None + if args.preferences_sketch != None: + preferences_fqfn = os.path.join( + os.path.dirname(source_globals_h_fqfn), + os.path.normpath(args.preferences_sketch)) + else: + if args.preferences_file != None: + preferences_fqfn = args.preferences_file + elif args.preferences_env != None: + preferences_fqfn = args.preferences_env + else: + ide_path = runtime_ide_path + + if preferences_fqfn != None: + preferences_fqfn = os.path.normpath(preferences_fqfn) + root = False + if 'Windows' == platform.system(): + if preferences_fqfn[1:2] == ':\\': + root = True + else: + if preferences_fqfn[0] == '/': + root = True + if not root: + if preferences_fqfn[0] != '~': + preferences_fqfn = os.path.join("~", preferences_fqfn) + preferences_fqfn = os.path.expanduser(preferences_fqfn) + print_dbg(f"determine_cache_state: preferences_fqfn: {preferences_fqfn}") + + try: + caching_enabled = check_preferences_txt(ide_path, preferences_fqfn) + except UserWarning: + if os.path.exists(source_globals_h_fqfn): + caching_enabled = None + print_err(f" runtime_ide_version: {args.runtime_ide_version}") + print_err(f" This must be resolved to use '{globals_name}'") + print_err(f" Read more at {docs_url}") + else: + # We can quietly ignore the problem because we are not needed. + caching_enabled = True + + return caching_enabled + + +""" +TODO + +aggressive caching workaround +========== ======= ========== +The question needs to be asked, is it a good idea? +With all this effort to aid in determining the cache state, it is rendered +usless when arduino command line switches are used that contradict our +settings. + +Sort out which of these are imperfect solutions should stay in + +Possible options for handling problems caused by: + ./arduino --preferences-file other-preferences.txt + ./arduino --pref compiler.cache_core=false + +--cache_core +--no_cache_core +--preferences_file (relative to IDE or full path) +--preferences_sketch (default looks for preferences.txt or specify path relative to sketch folder) +--preferences_env, python docs say "Availability: most flavors of Unix, Windows." + + export ARDUINO15_PREFERENCES_FILE=$(realpath other-name-than-default-preferences.txt ) + ./arduino --preferences-file other-name-than-default-preferences.txt + + platform.local.txt: mkbuildoptglobals.extra_flags=--preferences_env + + Tested with: + export ARDUINO15_PREFERENCES_FILE=$(realpath ~/projects/arduino/arduino-1.8.19/portable/preferences.txt) + ~/projects/arduino/arduino-1.8.18/arduino + + + Future Issues + * "--preferences-file" does not work for Arduino IDE 2.0, they plan to address at a future release + * Arduino IDE 2.0 does not support portable, they plan to address at a future release + +""" + + +def check_env(env): + system = platform.system() + # From the docs: + # Availability: most flavors of Unix, Windows. + # “Availability: Unix” are supported on macOS + # Because of the soft commitment, I used "help=argparse.SUPPRESS" to keep + # the claim out of the help. The unavailable case is untested. + val = os.getenv(env) + if val == None: + if "Linux" == system or "Windows" == system: + raise argparse.ArgumentTypeError(f'Missing environment variable: {env}') + else: + # OS/Library limitation + raise argparse.ArgumentTypeError('Not supported') + return val + + +def parse_args(): + extra_txt = '''\ + Use platform.local.txt 'mkbuildoptglobals.extra_flags=...' to supply override options: + --cache_core | --no_cache_core | --preferences_file PREFERENCES_FILE | ... + + more help at {} + '''.format(docs_url) + parser = argparse.ArgumentParser( + description='Prebuild processing for globals.h and build.opt file', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(extra_txt)) + parser.add_argument('runtime_ide_path', help='Runtime IDE path, {runtime.ide.path}') + parser.add_argument('runtime_ide_version', type=int, help='Runtime IDE Version, {runtime.ide.version}') + parser.add_argument('build_path', help='Build path, {build.path}') + parser.add_argument('build_opt_fqfn', help="Build FQFN to build.opt") + parser.add_argument('source_globals_h_fqfn', help="Source FQFN Sketch.ino.globals.h") + parser.add_argument('commonhfile_fqfn', help="Core Source FQFN CommonHFile.h") + parser.add_argument('--debug', action='store_true', required=False, default=False) + parser.add_argument('--ci', action='store_true', required=False, default=False) + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('--cache_core', action='store_true', default=None, help='Assume a "compiler.cache_core" value of true') + group.add_argument('--no_cache_core', dest='cache_core', action='store_false', help='Assume a "compiler.cache_core" value of false') + group.add_argument('--preferences_file', help='Full path to preferences file') + group.add_argument('--preferences_sketch', nargs='?', action='store', const="preferences.txt", help='Sketch relative path to preferences file') + # Since the docs say most versions of Windows and Linux support the os.getenv method, suppress the help message. + group.add_argument('--preferences_env', nargs='?', action='store', type=check_env, const="ARDUINO15_PREFERENCES_FILE", help=argparse.SUPPRESS) + # ..., help='Use environment variable for path to preferences file') + return parser.parse_args() + # ref epilog, https://stackoverflow.com/a/50021771 + # ref nargs='*'', https://stackoverflow.com/a/4480202 + # ref no '--n' parameter, https://stackoverflow.com/a/21998252 + +def main(): + global build_opt_signature + global docs_url + global debug_enabled + num_include_lines = 1 + + args = parse_args() + debug_enabled = args.debug + runtime_ide_path = os.path.normpath(args.runtime_ide_path) + build_path = os.path.normpath(args.build_path) + build_opt_fqfn = os.path.normpath(args.build_opt_fqfn) + source_globals_h_fqfn = os.path.normpath(args.source_globals_h_fqfn) + commonhfile_fqfn = os.path.normpath(args.commonhfile_fqfn) + + globals_name = os.path.basename(source_globals_h_fqfn) + build_path_core, build_opt_name = os.path.split(build_opt_fqfn) + globals_h_fqfn = os.path.join(build_path_core, globals_name) + + print_dbg(f"runtime_ide_path: {runtime_ide_path}") + print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") + print_dbg(f"build_path: {build_path}") + print_dbg(f"build_opt_fqfn: {build_opt_fqfn}") + print_dbg(f"source_globals_h_fqfn: {source_globals_h_fqfn}") + print_dbg(f"commonhfile_fqfn: {commonhfile_fqfn}") + print_dbg(f"globals_name: {globals_name}") + print_dbg(f"build_path_core: {build_path_core}") + print_dbg(f"globals_h_fqfn: {globals_h_fqfn}") + + if args.ci: + # Requires CommonHFile.h to never be checked in. + if os.path.exists(commonhfile_fqfn): + first_time = False + else: + first_time = True + else: + first_time = discover_1st_time_run(build_path) + if first_time: + print_dbg("First run since Arduino IDE started.") + + use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) + if use_aggressive_caching_workaround == None: + # Specific rrror messages already buffered + handle_error(1) + + print_dbg(f"first_time: {first_time}") + print_dbg(f"use_aggressive_caching_workaround: {use_aggressive_caching_workaround}") + + if first_time or \ + not use_aggressive_caching_workaround or \ + not os.path.exists(commonhfile_fqfn): + enable_override(False, commonhfile_fqfn) + + # A future timestamp on commonhfile_fqfn will cause everything to + # rebuild. This occurred during development and may happen after + # changing the system time. + if time.time_ns() < os.stat(commonhfile_fqfn).st_mtime_ns: + touch(commonhfile_fqfn) + print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}") + + if not os.path.exists(build_path_core): + os.makedirs(build_path_core) + print_msg("Clean build, created dir " + build_path_core) + + if os.path.exists(source_globals_h_fqfn): + print_msg("Using global include from " + source_globals_h_fqfn) + + copy_create_build_file(source_globals_h_fqfn, globals_h_fqfn) + + # globals_h_fqfn timestamp was only updated if the source changed. This + # controls the rebuild on change. We can always extract a new build.opt + # w/o triggering a needless rebuild. + embedded_options = extract_create_build_opt_file(globals_h_fqfn, globals_name, build_opt_fqfn) + + if use_aggressive_caching_workaround: + # commonhfile_fqfn encodes the following information + # 1. When touched, it causes a rebuild of core.a + # 2. When file size is non-zero, it indicates we are using the + # aggressive cache workaround. The workaround is set to true + # (active) when we discover a non-zero length global .h file in + # any sketch. The aggressive workaround is cleared on the 1ST + # compile by the Arduino IDE after starting. + # 3. When the timestamp matches the build copy of globals.h + # (globals_h_fqfn), we know one two things: + # * The cached core.a matches up to the current build.opt and + # globals.h. The current sketch owns the cached copy of core.a. + # * globals.h has not changed, and no need to rebuild core.a + # 4. When core.a's timestamp does not match the build copy of + # the global .h file, we only know we need to rebuild core.a, and + # that is enough. + # + # When the sketch build has a "Sketch.ino.globals.h" file in the + # build tree that exactly matches the timestamp of "CommonHFile.h" + # in the platform source tree, it owns the core.a cache copy. If + # not, or "Sketch.ino.globals.h" has changed, rebuild core. + # A non-zero file size for commonhfile_fqfn, means we have seen a + # globals.h file before and workaround is active. + if debug_enabled: + print_dbg("Timestamps at start of check aggressive caching workaround") + ts = os.stat(globals_h_fqfn) + print_dbg(f" globals_h_fqfn ns_stamp = {ts.st_mtime_ns}") + print_dbg(f" getmtime(globals_h_fqfn) {os.path.getmtime(globals_h_fqfn)}") + ts = os.stat(commonhfile_fqfn) + print_dbg(f" commonhfile_fqfn ns_stamp = {ts.st_mtime_ns}") + print_dbg(f" getmtime(commonhfile_fqfn) {os.path.getmtime(commonhfile_fqfn)}") + + if os.path.getsize(commonhfile_fqfn): + if (os.path.getmtime(globals_h_fqfn) != os.path.getmtime(commonhfile_fqfn)): + # Need to rebuild core.a + # touching commonhfile_fqfn in the source core tree will cause rebuild. + # Looks like touching or writing unrelated files in the source core tree will cause rebuild. + synchronous_touch(globals_h_fqfn, commonhfile_fqfn) + print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") + else: + print_dbg(f"Using old cached 'core.a'") + elif os.path.getsize(globals_h_fqfn): + enable_override(True, commonhfile_fqfn) + synchronous_touch(globals_h_fqfn, commonhfile_fqfn) + print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") + else: + print_dbg(f"Workaround not active/needed") + + add_include_line(build_opt_fqfn, commonhfile_fqfn) + add_include_line(build_opt_fqfn, globals_h_fqfn) + + # Provide context help for build option support. + source_build_opt_h_fqfn = os.path.join(os.path.dirname(source_globals_h_fqfn), "build_opt.h") + if os.path.exists(source_build_opt_h_fqfn) and not embedded_options: + print_err("Build options file '" + source_build_opt_h_fqfn + "' not supported.") + print_err(" Add build option content to '" + source_globals_h_fqfn + "'.") + print_err(" Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") + print_err(" Read more at " + docs_url) + elif os.path.exists(source_globals_h_fqfn): + if not embedded_options: + print_msg("Tip: Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") + print_msg(" Read more at " + docs_url) + else: + print_msg("Note: optional global include file '" + source_globals_h_fqfn + "' does not exist.") + print_msg(" Read more at " + docs_url) + + handle_error(0) # commit print buffer + +if __name__ == '__main__': + sys.exit(main())