From 44ef1ee1204a58c5708b8aa633923748cd7ec31c Mon Sep 17 00:00:00 2001 From: Brian Park Date: Fri, 16 Mar 2018 15:51:22 -0700 Subject: [PATCH 1/4] Implement a time out in TestRunner (default 10 seconds), to prevent testing() test cases which accidentally run forever --- src/aunit/Test.cpp | 2 +- src/aunit/Test.h | 4 +++ src/aunit/TestRunner.cpp | 58 ++++++++++++++++++++++++++--------- src/aunit/TestRunner.h | 26 ++++++++++++++++ tests/AUnitTest/AUnitTest.ino | 15 +++++++++ 5 files changed, 90 insertions(+), 15 deletions(-) diff --git a/src/aunit/Test.cpp b/src/aunit/Test.cpp index 120e89d..6c4430e 100644 --- a/src/aunit/Test.cpp +++ b/src/aunit/Test.cpp @@ -60,7 +60,7 @@ void Test::setPassOrFail(bool ok) { // Insert the current test case into the singly linked list, sorted by // getName(). This is an O(N^2) algorithm, but should be good enough for // small N ~= 100. If N becomes bigger than that, it's probably better to insert -// using an O(N) alorithm, then sort the elements later in TestRunner::run(). +// using an O(N) algorithm, then sort the elements later in TestRunner::run(). // Also, we don't increment a static counter here, because that would introduce // another static initialization ordering problem. void Test::insert() { diff --git a/src/aunit/Test.h b/src/aunit/Test.h index cd8d0b2..9220174 100644 --- a/src/aunit/Test.h +++ b/src/aunit/Test.h @@ -66,6 +66,7 @@ class Test { static const uint8_t kStatusPassed = 2; static const uint8_t kStatusFailed = 3; static const uint8_t kStatusSkipped = 4; + static const uint8_t kStatusExpired = 5; /** * Get the pointer to the root pointer. Implemented as a function static so @@ -123,6 +124,9 @@ class Test { /** Mark the test as skipped. */ void skip() { mStatus = kStatusSkipped; } + /** Mark the test as expired (i.e. timed out). */ + void expire() { mStatus = kStatusExpired; } + protected: /** Mark the test as failed. */ void fail() { mStatus = kStatusFailed; } diff --git a/src/aunit/TestRunner.cpp b/src/aunit/TestRunner.cpp index 51fae66..587eb6f 100644 --- a/src/aunit/TestRunner.cpp +++ b/src/aunit/TestRunner.cpp @@ -72,7 +72,8 @@ TestRunner::TestRunner(): mCount(0), mPassedCount(0), mFailedCount(0), - mSkippedCount(0) {} + mSkippedCount(0), + mTimeout(kTimeoutDefault) {} void TestRunner::runTest() { setupRunner(); @@ -103,15 +104,29 @@ void TestRunner::runTest() { } break; case Test::kStatusSetup: - (*mCurrent)->loop(); - - // If test status is unresolved (i.e. still in kStatusSetup state) after - // loop(), then this is a continuous testing() test case, so skip to the - // next test. Otherwise, stay on the current test so that the next - // iteration can resolve the current test. - if ((*mCurrent)->getStatus() == Test::kStatusSetup) { - // skip to the next one, but keep current test in the list - mCurrent = (*mCurrent)->getNext(); + { + // Check for timeout. mTimeout == 0 means infinite timeout. + // NOTE: It feels like this code should go into the Test::loop() method + // (like the extra bit of code in TestOnce::loop()) because it seems + // like we could want the timeout to be configurable on a case by case + // basis. This would cause the testing() code to move down into a new + // again() virtual method dispatched from Test::loop(), analogous to + // once(). But let's keep the code here for now. + unsigned long now = millis(); + if (mTimeout > 0 && now >= mStartTime + mTimeout) { + (*mCurrent)->expire(); + } else { + (*mCurrent)->loop(); + + // If test status is unresolved (i.e. still in kStatusSetup state) + // after loop(), then this is a continuous testing() test case, so + // skip to the next test. Otherwise, stay on the current test so that + // the next iteration of runTest() can resolve the current test. + if ((*mCurrent)->getStatus() == Test::kStatusSetup) { + // skip to the next one, but keep current test in the list + mCurrent = (*mCurrent)->getNext(); + } + } } break; case Test::kStatusSkipped: @@ -132,6 +147,12 @@ void TestRunner::runTest() { // skip to the next one by taking current test out of the list *mCurrent = *(*mCurrent)->getNext(); break; + case Test::kStatusExpired: + mExpiredCount++; + resolveTest((*mCurrent)); + // skip to the next one by taking current test out of the list + *mCurrent = *(*mCurrent)->getNext(); + break; } } @@ -140,6 +161,7 @@ void TestRunner::setupRunner() { mIsSetup = true; mCount = countTests(); mCurrent = Test::getRoot(); + mStartTime = millis(); } } @@ -167,19 +189,23 @@ void TestRunner::resolveTest(Test* testCase) { Printer::getPrinter()->println(F(" failed.")); } else if (testCase->getStatus ()== Test::kStatusPassed) { Printer::getPrinter()->println(F(" passed.")); + } else if (testCase->getStatus ()== Test::kStatusExpired) { + Printer::getPrinter()->println(F(" timed out.")); } } void TestRunner::resolveRun() { if (!isVerbosity(Verbosity::kTestRunSummary)) return; - Printer::getPrinter()->print(F("Test summary: ")); + Printer::getPrinter()->print(F("TestRunner summary: ")); Printer::getPrinter()->print(mPassedCount); Printer::getPrinter()->print(F(" passed, ")); Printer::getPrinter()->print(mFailedCount); - Printer::getPrinter()->print(F(" failed, and ")); + Printer::getPrinter()->print(F(" failed, ")); Printer::getPrinter()->print(mSkippedCount); - Printer::getPrinter()->print(F(" skipped, out of ")); + Printer::getPrinter()->print(F(" skipped, ")); + Printer::getPrinter()->print(mExpiredCount); + Printer::getPrinter()->print(F(" timed out, out of ")); Printer::getPrinter()->print(mCount); Printer::getPrinter()->println(F(" test(s).")); @@ -189,7 +215,7 @@ void TestRunner::resolveRun() { void TestRunner::listTests() { setupRunner(); - Printer::getPrinter()->print("Test count: "); + Printer::getPrinter()->print("TestRunner test count: "); Printer::getPrinter()->println(mCount); for (Test** p = Test::getRoot(); (*p) != nullptr; p = (*p)->getNext()) { Printer::getPrinter()->print(F("Test ")); @@ -198,4 +224,8 @@ void TestRunner::listTests() { } } +void TestRunner::setRunnerTimeout(unsigned long timeout) { + mTimeout = timeout; +} + } diff --git a/src/aunit/TestRunner.h b/src/aunit/TestRunner.h index 18bb717..c6da95f 100644 --- a/src/aunit/TestRunner.h +++ b/src/aunit/TestRunner.h @@ -78,7 +78,27 @@ class TestRunner { /** Set the pass/fail status of the current test. */ static void setPassOrFail(bool ok) { getRunner()->setTestPassOrFail(ok); } + /** + * Set test runner timeout across all tests, in millis. Set to 0 for + * infinite timeout. Useful for preventing testing() test cases that never + * end. This a timeout for the TestRunner itself, not for individual tests. + * + * It might be usefult to allow timeouts on a per-test basis, but I'm + * reluctant to do that at the Test class level because it would add extra + * 4 bytes (maybe 2 if we allowed a small maximum) of static memory per + * test case. The solution might be to allow the end-user to choose to pay + * for this extra cost if they want to. I think the right way to do that is + * to add support for test fixtures which is something on the back of my + * mind. + */ + static void setTimeout(long millis) { + getRunner()->setRunnerTimeout(millis); + } + private: + // 10 second timeout for the runner + static const unsigned long kTimeoutDefault = 10000; + /** Return the singleton TestRunner. */ static TestRunner* getRunner(); @@ -119,6 +139,9 @@ class TestRunner { /** Set the pass/fail status of the current test. */ void setTestPassOrFail(bool ok) { (*mCurrent)->setPassOrFail(ok); } + /** Set the test runner timeout. */ + void setRunnerTimeout(unsigned long timeout); + // The current test case is represented by a pointer to a pointer. This // allows treating the root node the same as all the other nodes, and // simplifies the code traversing the singly-linked list significantly. @@ -131,6 +154,9 @@ class TestRunner { uint16_t mPassedCount; uint16_t mFailedCount; uint16_t mSkippedCount; + uint16_t mExpiredCount;; + unsigned long mTimeout; + unsigned long mStartTime; }; } diff --git a/tests/AUnitTest/AUnitTest.ino b/tests/AUnitTest/AUnitTest.ino index bd31596..fc3c130 100644 --- a/tests/AUnitTest/AUnitTest.ino +++ b/tests/AUnitTest/AUnitTest.ino @@ -353,6 +353,16 @@ testing(looping_pass) { } } +testing(looping_until) { + static unsigned long startTime = millis(); + + // complete the test in 20 secons. + unsigned long now = millis(); + if (now >= startTime + 20000) { + pass(); + } +} + void setup() { Serial.begin(74880); // 74880 is the default for some ESP8266 boards while (! Serial); // Wait until Serial is ready - Leonardo @@ -361,6 +371,11 @@ void setup() { //TestRunner::setVerbosity(Verbosity::kAll); TestRunner::exclude("looping_f*"); TestRunner::list(); + + // If set to something really small, like 1, all tests are incomplete. + // If set to 0, infinite timeout, some testing() may accidentally run forever. + // Default is 10000 ms. + //TestRunner::setTimeout(25000); #else //Test::min_verbosity = TEST_VERBOSITY_ALL; Test::exclude("looping_f*"); From 9568d424a917215f27f47b7b60cd15ab65e9631f Mon Sep 17 00:00:00 2001 From: Brian Park Date: Fri, 16 Mar 2018 15:53:00 -0700 Subject: [PATCH 2/4] README.md: huge refactoring, add info about TestRunner timeouts, scatter information about ArduinoUnit compatibility through the Usage section, removed Migrating section which contains much duplicated info --- README.md | 567 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 314 insertions(+), 253 deletions(-) diff --git a/README.md b/README.md index ac5d0b8..c7ad091 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,9 @@ on the venerable [ArduinoUnit](https://github.com/mmurdoch/arduinounit) framework. It is almost a drop-in replacement for the API implemented by ArduinoUnit version 2.2. Just like ArduinoUnit, the unit tests run directly on -the microcontrollers themselves, not on emulators or simulators. +the microcontrollers themselves, not on emulators or simulators. The test +results are printed on the `Serial` object by default, but can be redirected to +another `Print` object. AUnit was created to solve 2 problems with ArduinoUnit: * ArduinoUnit consumes too much flash memory on an AVR platform (e.g. @@ -17,18 +19,20 @@ AUnit was created to solve 2 problems with ArduinoUnit: [ArduinoUni#57](https://github.com/mmurdoch/arduinounit/pull/57), and [ArduinoUni#55](https://github.com/mmurdoch/arduinounit/issues/55)). - In contrast: * AUnit consumes as much as 66% *less* flash memory than ArduinoUnit on the AVR platform. On Teensy-ARM, the savings can be as much as 30%. * AUnit has been tested on AVR, Teensy-ARM and ESP8266. +### Supported or Compatible Features + For basic unit tests written using ArduinoUnit, only two changes are required to convert to AUnit: * `#include ` -> `#include ` * `Test::run()` -> `aunit::TestRunner::run()` -Most of the frequently used macros are compatible between ArduinoUnit and AUnit: +Almost all of the frequently used macros are compatible between ArduinoUnit and +AUnit: * `test()` * `testing()` * `assertXxx()` @@ -37,37 +41,56 @@ AUnit supports exclude and include filters: * `TestRunner::exclude()` * `TestRunner::include()` -Currently, only a single `*` wildcard is supported and it must occur at the end -if present. +### Missing Features + +Here are the features which have not been ported over from ArduinoUnit: + +* ArduinoUnit supports multiple `*` wildcards in its `exclude()` and `include()` + methods. AUnit supports only a single `*` wildcard and it must occur at the + end if present. +* Various "Meta Assertions" from ArduinoUnit (i.e. `checkTestXxx()` and + `assertTestXxx()`) have not been implemented. + +### Added Features -Various "Meta Assertions" from ArduinoUnit have not been implemented yet -(see *Migrating from ArduinoUnit to AUnit* section below). +Here are some features in AUnit, not available in ArduinoUnit: -(**Beta Status**: Although this library has been extensively tested by me, and I -converted my [AceButton](https://github.com/bxparks/AceButton) library to use -it, I consider it currently in "beta stage" until more users have tested it.) +* The `TestRunner` supports a configurable timeout parameter which + can prevent `testing()` test cases from running forever. +* AUnit works on the ESP8266 platform. + +### Beta Status + +Although this library has been extensively tested by me, and I converted my +[AceButton](https://github.com/bxparks/AceButton) library to use it, I consider +it currently in "beta stage" until more users have tested it. ## Installation -The library will be available in the Arduino IDE Library Manager eventually. +The library is available in the Arduino IDE Library Manager. Search for "unit +test" or "AUnit", select "AUnit", then click the "Install" button. -In the mean time, it can be installed by cloning the +The library can also be installed by cloning the [GitHub repository](https://github.com/bxparks/AUnit), then manually copying over the contents to the `./libraries` directory used by the Arduino IDE. (The result is a directory named `./libraries/AUnit`.) See the Preferences menu for the location of the Arduino sketch directory. -You may need to restart the Arduino IDE to let it see the new library. + +Using either installation method, you may need to restart the Arduino IDE to +pick up the new library. ## Usage +In this section, information about differences between AUnit and ArduinoUnit +will appear in a note marked by ***ArduinoUnit Compatibility***. + ### Header and Namespace To prevent name clashes with other libraries and code, all classes in the AUnit -library are defined in the `aunit` namespace. The user will normally interact -with only the `TestRunner` class. It can be reference with an explicit namespace +library are defined in the `aunit` namespace. The user will mostly interact with +the `TestRunner` class. It can be referenced with an explicit namespace qualifier (i.e. `aunit::TestRunner`), or we can use a `using` directive like this: - ``` #include using aunit::TestRunner; @@ -83,24 +106,49 @@ Similar to ArduinoUnit, many of the "functions" in this framework (e.g. in the global namespace, so it is usually not necessary to import the entire `aunit` namespace. +***ArduinoUnit Compatibility***: _I have found that the following macros are +useful during the transition:_ +``` +#define USE_AUNIT 1 + +#if USE_AUNIT == 1 +#include +#else +#include +#endif + +... + +void loop() { +#if USE_AUNIT == 1 + aunit::TestRunner::run(); +#else + Test::run(); +#endif +} +``` + ### Defining the Tests -The usage of **AUnit** is almost identical to **ArduinoUnit**. A test case is -defined by a fragment of code inside the `{ }` just after the `test()` or -`testing()` macros. The argument to these macros are the name of the test case. -(The name is available within the test code using the `Test::getName()` -method). The `test()` and `testing()` macros use the name to generate a subclass -whose parents are `aunit::TestOnce` and `aunit::Test` respectively. +The usage of **AUnit** is basically identical to **ArduinoUnit**. The following +macros are used to create a subclass of one of the two test classes (`Test` or +`TestOnce`): + +* `test()` +* `testing()` + +and the code in `{ }` following these macros becomes the body of the `looping()` +or `once()` methods of the two base test classes. -The macros also generate code to create an instance of the subclass. -The code following after the `test()` and `testing()` macros becomes -the body of the virtual `TestOnce::once()` and `Test::loop` methods -(respectively). +The argument to these macros are the name of the test case, and is used to +generate a name for the subclass. (The name is available within the test code +using the `Test::getName()` method). The macros also generate code to create an +global instance of the subclass, which are static initialized by C++. -When the instance of the test case is statically initialized, it adds itself to -a linked list. The root of that singly-linked list is given by -`Test::getRoot()`. The `TestRunner::run()` method traverses the linked list, -executing each test case until it passes, fails or is skipped. +During static initialization, the constructor of the object adds itself to an +internal list. The root of that list is given by `Test::getRoot()`. The +`TestRunner::run()` method traverses the linked list, executing each test case +until it passes, fails or is skipped. Here is a rough outline of an AUnit unit test sketch: @@ -140,14 +188,10 @@ void loop() { } ``` -### Supported Macros +***ArduinoUnit Compatibility***: _The basic structure of the +unit test is identical to ArduinoUnit._ -These are identical to ArduinoUnit: - -* `test()` -* `testing()` - -### Supported Assertions +### Binary Assertions Inside the `test()` and `testing()` macros, the following assertions are available. These are essentially identical to ArduinoUnit: @@ -159,29 +203,8 @@ are available. These are essentially identical to ArduinoUnit: * `assertLessOrEqual(a, b)` * `assertMoreOrEqual(a, b)` -The type inference logic of two `(a, b)` arguments in the `assertXxx(a, b)` is -slightly different than ArduinoUnit. For non-String types, `a` and `b` must have -the same type, after the usual implicit type converisons. For example, the -following implicit conversions will occur: -* `signed char` -> `int` -* `unsigned char` -> `int` -* `short` -> `int` -* `unsigned short` -> `int` or `unsigned int` (depending on `sizeof(int)`) -* `char*` -> `const char*`. -* `char[N]` -> `const char*` -* `float` -> `double` - -(Note that `char`, `signed char`, and `unsigned char` are 3 distinct types in -C++.) - -Mixing the parameter types will often produce compiler errors. See comments -and solutions in the *Migrating from ArduinoUnit to AUnit* section below. - -For the 3 string types (`char*`, `String`, and `__FlashStringHelper*`), -all 9 combinatorial mixes are supported. - -In summary, the following types of `(a, b)` are defined for the various -`assertXxx()` macros: +The following overloaded types for the various `assertXxx()` macros are +defined: * `(bool, bool)` * `(char, char)` @@ -200,37 +223,150 @@ In summary, the following types of `(a, b)` are defined for the various * `(const __FlashStringHelper*, const String&)` * `(const __FlashStringHelper*, const __FlashStringHelper*)` -The following boolean asserts are also available, just like ArduinoUnit: +As you can see, all 9 combinations of the 3 string types (`char*`, `String`, and +`__FlashStringHelper*`) are supported. + +Additionally, the usual C++ implicit type conversion and function overloading +matching algorithms apply. For example, the conversions will occur: + +* `signed char` -> `int` +* `unsigned char` -> `int` +* `short` -> `int` +* `unsigned short` -> `int` or `unsigned int` (depending on `sizeof(int)`) +* `char*` -> `const char*`. +* `char[N]` -> `const char*` +* `float` -> `double` + +Note that `char`, `signed char`, and `unsigned char` are 3 distinct types in +C++, so a `(char, char)` will match exactly to one of the `assertXxx()` +methods. + +***ArduinoUnit Compatibility***: +_The names of the macros are identical. However, the +type inference logic of two `(a, b)` arguments in the `assertXxx(a, b)` is +slightly different. ArduinoUnit allows the two parameters to be slightly +different types, at the expense of a compiler warning. In AUnit, the +warning becomes a compiler error. See below._ + +#### Assertion Parameters Must Match Types + +In ArduinoUnit, the `assertXxx()` macros could be slightly different types, for +example: +``` +unsigned int uintValue = 5; +assertEqual(5, uintValue); +``` + +If the compiler warnings are enabled, a warning from the compiler is printed: + +``` +../ArduinoUnit/src/ArduinoUnitUtility/Compare.h:17:28: warning: + comparison between signed and unsigned integer expressions [-Wsign-compare] + return (!(a, unsigned int&)' is ambiguous + if (!aunit::assertion(__FILE__,__LINE__,(arg1),opName,op,(arg2)))\ +... +``` +The compiler cannot find an appropriate overloaded version of `assertEqual()`. + +The solution is to make the parameters the same type: +``` +assertEqual(5U, uintValue); +``` + +On the AVR platform, both a (short) and (int) are 16-bit types, so the following +will produce a compiler error: +``` +unsigned short ushortValue = 5; +assertEqual(5U, ushortValue); +``` +But on Teensy-ARM and ESP8266, a 16-bit (short) can be promoted to a 32-bit +(int) without loss of precision, so the above will compile just fine. For +portability, the following should be used on all platforms: +``` +unsigned short ushortValue = 5; +assertEqual((unsigned short) 5, ushortValue); +``` + +The integer type promotion rules and function overload matching rules can be +difficult to remember (and sometimes difficult to understand). The best way to +avoid these compiler errors is to make sure that the assertion parameter types +are identical, potentially using explicit casting. + +### Boolean Assertions + +The following boolean asserts are also available: * `assertTrue(condition)` * `assertFalse(condition)` -### Supported Methods in a Test Case +***ArduinoUnit Compatibility***: _These are identical to ArduinoUnit._ -These are identical to the equivalent methods in ArduinoUnit, and are available -in a `test()` or `testing()` macro: +### Meta Assertions -* `pass()` -* `fail()` -* `skip()` +***ArduinoUnit Compatibility***: _Not implemented in AUnit._ -### Supported Methods on Test Class +The following methods from ArduinoUnit are not implemented: -These are identical to ArduinoUnit: +* `checkTestDone(name)` +* `checkTestNotDone(name)` +* `checkTestPass(name)` +* `checkTestNotPass(name)` +* `checkTestFail(name)` +* `checkTestNotFail(name)` +* `checkTestSkip(name)` +* `checkTestNotSkip(name)` +* `assertTestDone(name)` +* `assertTestNotDone(name)` +* `assertTestPass(name)` +* `assertTestNotPass(name)` +* `assertTestFail(name)` +* `assertTestNotFail(name)` +* `assertTestSkip(name)` +* `assertTestNotSkip(name)` + +***ArduinoUnit Compatibility***: _Not implemented in AUnit._ + +### Status Indicator Methods + +These methods can be used inside a `test()` or `testing()` macro +to indicate whether the test has passed or failed (or reached some other +status reason). + +* `pass()` - test passed +* `fail()` - test failed +* `skip()` - test skipped +* `expire()` - test timed out + +***ArduinoUnit Compatibility***: _`expire()` is available only in AUnit._ + +### Overridable Methods + +The following methods are defined at the `Test` base class level: * `setup()` * `loop()` * `once()` +***ArduinoUnit Compatibility***: _These are identifcal to ArduinoUnit._ + ### Running the Tests -Similar to ArduinioUnit, we run the test cases in the global `loop()` method by -calling `TestRunner::run()`. Each call to the `run()` method causes one test -case to run and be resolved. The next call to `run()` executes the next test -case. This design allows the `loop()` method to perform a small amount of work -and return periodically to allow the system to perform some actions. On some -systems, such as the ESP8266, an error is generated if `loop()` takes too much -CPU time. +We run the test cases in the global `loop()` method by calling +`TestRunner::run()`. The tests are sorted according to the name of the test +given in the argument in the `test()` or `testing()` macro. + +Each call to the `run()` method causes one test case to run and be resolved. The +next call to `run()` executes the next test case. This design allows the +`loop()` method to perform a small amount of work and return periodically to +allow the system to perform some actions. On some systems, such as the ESP8266, +an error is generated if `loop()` takes too much CPU time. ``` ... @@ -239,13 +375,19 @@ void loop() { } ``` +***ArduinoUnit Compatibility***: _This is equivalent to called `Test::run()` in +ArduinoUnit. AUnit sorts the tests in the same way as ArduinoUnit. In +ArduinoUnit, each call to `Test::run()` will process the entire list of +currently active test cases. In AUnit, each call to `TestRunner::run()` performs +only a single test case, then returns._ + ### Excluding and Including Test Cases We can `exclude()` or `include()` test cases using a pattern match, just like ArduinoUnit. The names are slightly different: -* `TestRunner::exclude()` (equivalent to `Test::exclude()` -* `TestRunner::include()` (equivalent to `Test::include()` +* `TestRunner::exclude()` +* `TestRunner::include()` These methods are called from the global `setup()` method: @@ -257,15 +399,17 @@ void setup() { } ``` -The `TestRunner::exclude()` and `TestRunner::include()` methods in AUnit -are not as powerful as the ones provided by ArduinoUnit. AUnit supports only a -single wildcard character `*` and that character can appear only at the end if -it is present. For example, the following are accepted: +***ArduinoUnit Compatibility***: +_The equivalent versions in ArduinoUnit are `Test::exclude()` and +`Test::include()` The matching algorithm in AUnit is not as powerful as one in +ArduinoUnit. AUnit supports only a single wildcard character `*` and that +character can appear only at the end if it is present. For example, the +following are accepted:_ * `TestRunner::exclude("*");` -* `TestRunner::exclude("f*");` -* `TestRunner::include("flash_*");` -* `TestRunner::exclude("looping*");` +* `TestRunner::include("f*");` +* `TestRunner::exclude("flash_*");` +* `TestRunner::include("looping*");` * `TestRunner::include("flashTest");` ### Output Printer @@ -290,9 +434,9 @@ void loop() { } ``` -This is the equivalent of the `Test::out` static member variable in -ArduinoUnit. In AUnit member variables are not exposed, so changes must be -made through the `TestRunner::setPrinter()` method. +***ArduinoUnit Compatibility***: +_This is the equivalent of the `Test::out` static member variable in +ArduinoUnit._ ### Controlling the Verbosity @@ -329,99 +473,19 @@ static constants of the `Verbosity` utility class: * `Verbosity::kAssertionPassed` * `Verbosity::kAssertionAll` * `Verbosity::kDefault`, equivalent to setting the following - * `Verbosity::kAssertionFailed` - * `Verbosity::kTestFailed` - * `Verbosity::kTestPassed` - * `Verbosity::kTestSkipped` - * `Verbosity::kTestRunSummary` + * `(Verbosity::kAssertionFailed | Verbosity::kTestAll)` * `Verbosity::kAll` - enables all messages * `Verbosity::kNone` - disables all messages -### Line Number Mismatch - -AUnit suffers from the same compiler/preprocessor bug as ArduinoUnit that causes -the built-in `__LINE__` macro to be off by one. The solution is to add: -``` -#line 2 {file.ino} -``` -as the first line of a unit test sketch. - -## Migrating from ArduinoUnit to AUnit - -### Header - -Use -``` -#include -``` -instead of -``` -#include -``` - -### Test Runner - -The `Test::run()` method has been moved to a new `TestRunner` class. Use -``` -aunit::TestRunner::run(); -``` -instead of -``` -Test::run(); -``` - -### Compile Time Selection - -I have found that the following macros are useful during the transition: - -``` -#define USE_AUNIT 1 - -#if USE_AUNIT == 1 -#include -#else -#include -#endif - -... - -void loop() { -#if USE_AUNIT == 1 -aunit::TestRunner::run(); -#else -Test::run(); -#endif -} -``` - -### Printer - -The `Test::out` static variable can be set using the static method on -`TestRunner`. Use - -``` -TestRunner::setPrinter(&Serial1); -``` -instead of -``` -Test::out = &Serial1; -``` - -(The current `Print` object can be accessed through -`aunit::Printer::getPrinter()` but I don't expect this to be needed often.) - -### Verbosity - -The following ArduinoUnit variables do not exist: +***ArduinoUnit Compatibility***: +_The following ArduinoUnit variables do not exist:_` * `Test::verbosity` * `Test::min_verbosity` * `Test::max_verbosity` -Verbosity can be set only at the `TestRunner` level. Verbosity cannot be set at -the test case (i.e. `Test` or `TestOnce` class) level individually. There is no -"min" or "max" verbosity level. Each type of message is controlled by a bit -flag. The bit flag can be set using `TestRunner::setVerbosity()`. The bit field -constants have slightly different names: +_In AUnit, verbosity can be set only at the `TestRunner` level. Verbosity cannot +be set at the test case (i.e. `Test` or `TestOnce` class) level individually. +The bit field constants have slightly different names:_ * `TEST_VERBOSITY_TESTS_SUMMARY` -> `Verbosity::kTestRunSummary` * `TEST_VERBOSITY_TESTS_FAILED` -> `Verbosity::kTestFailed` @@ -435,110 +499,107 @@ constants have slightly different names: * `TEST_VERBOSITY_NONE` -> `Verbosity::kNone` * {no equivalent} <- `Verbosity::kDefault` -### Missing ArduinoUnit Features - -The following methods from ArduinoUnit are not yet implemented: - -* `checkTestDone(name)` -* `checkTestNotDone(name)` -* `checkTestPass(name)` -* `checkTestNotPass(name)` -* `checkTestFail(name)` -* `checkTestNotFail(name)` -* `checkTestSkip(name)` -* `checkTestNotSkip(name)` -* `assertTestDone(name)` -* `assertTestNotDone(name)` -* `assertTestPass(name)` -* `assertTestNotPass(name)` -* `assertTestFail(name)` -* `assertTestNotFail(name)` -* `assertTestSkip(name)` -* `assertTestNotSkip(name)` +### Line Number Mismatch -### Smaller Test Runner Loop Chunks +AUnit suffers from the same compiler/preprocessor bug as ArduinoUnit that causes +the built-in `__LINE__` macro to be off by one. The solution is to add: +``` +#line 2 {file.ino} +``` +as the first line of a unit test sketch. -In ArduinoUnit, each call to `Test::run()` will process the entire list of -currently active test cases. In AUnit, each call to `TestRunner::run()` will -process just one test case and return. I chose to break up the -`TestRunner::run()` method into smaller pieces to allow the `loop()` method to -return to the system more frequently. This is especially important on the -ESP8266 platform where system must get some periodic CPU cycles to perform its -own tasks. +***ArduinoUnit Compatibility***: _This problem is identical to ArduinoUnit._ -### Assertion Parameters Omitted in Messages +### Assertion Message -The various `assertXxx()` macros in AUnit print a slightly shorter -message upon pass or fail. For example, if the assertion was: +The various `assertXxx()` macros in AUnit print a message upon pass or fail. For +example, if the assertion was: ``` int expected = 3; int counter = 4; assertEquals(expected, counter); ``` -ArduinoUnit captures the arguments of the `assertEqual()` macro -and prints: - +The error message (if enabled, which is the default) is: ``` -Assertion failed: (expected=3) == (counter=4), file AUnitTest.ino, line 134. +Assertion failed: (3) == (4), file AUnitTest.ino, line 134. ``` -AUnit omits the parameters to reduce flash memory space by about 33%: +***ArduinoUnit Compatibility***: +_ArduinoUnit captures the arguments of the `assertEqual()` macro +and prints:_ ``` -Assertion failed: (3) == (4), file AUnitTest.ino, line 134. +Assertion failed: (expected=3) == (counter=4), file AUnitTest.ino, line 134. ``` -### Assertion Parameters Must Match Types +_Each capture of the parameter string consumes flash memory space. If the unit +test has numerous `assertXxx()` statements, the flash memory cost is expensive. +AUnit omits the parameters to reduce flash memory space by about 33%_ -In ArduinoUnit, the `assertXxx()` macros could be slightly different types, for -example: -``` -unsigned int uintValue = 5; -assertEqual(5, uintValue); -``` +### Test Summary -If the compiler warnings are enabled, a warning from the compiler is printed: +As each test case finishes, the `TestRunner` prints out the summary of the test +case like this: ``` -../ArduinoUnit/src/ArduinoUnitUtility/Compare.h:17:28: warning: - comparison between signed and unsigned integer expressions [-Wsign-compare] - return (!(a, unsigned int&)' is ambiguous - if (!aunit::assertion(__FILE__,__LINE__,(arg1),opName,op,(arg2)))\ -... -``` -The compiler cannot find an appropriate overloaded version of `assertEqual()`. +***ArduinoUnit Compatibility***: _These are identifcal to ArduinoUnit, +except that the "timed out" status is new to AUnit. See Test Timeout +section below._ + +### Test Runner Summary + +At the end of the test run, the `TestRunner` prints out the summary +of all test cases, like this: -The solution is to make the parameters the same type: ``` -assertEqual(5U, uintValue); +TestRunner summary: 12 passed, 0 failed, 2 skipped, 1 timed out, out of 15 test(s). ``` -On the AVR platform, both a (short) and (int) are 16-bit types, so the following -will produce a compiler error: +***ArduinoUnit Compatibility***: _The message format is slightly different than +ArduinoUnit. I changed "Test summary" to "TestRunner summary" because the former +looks identical to the message that could have been printed by a `test(summary)` +test case. AUnit also adds information about tests which timed out. See below._ + +### Test Timeout + +***ArduinoUnit Compatibility***: _Only available in AUnit._ + +From my experience, it seems incredibly easy to write a `testing()` test case +which accidentally runs forever because the code forgets to call an explicit +`pass()`, `fail()` or `skip()`. + +The `TestRunner` in AUnit applies a time out value to all the test cases that it +runs. The default time out is 10000 milliseconds (10 seconds). Currently, the +time out value is global to all test cases, individual test time out values +cannot be set independently. If a test does not finish before that time, then +the test is marked as `timed out` (internally implemented by the +`Test::expire()` method) and a message is printed like this: ``` -unsigned short ushortValue = 5; -assertEqual(5U, ushortValue); +Test looping_until timed out. ``` -But on Teensy-ARM and ESP8266, a 16-bit (short) can be promoted to a 32-bit -(int) without loss of precision, so the above will compile just fine. For -portability, the following should be used on all platforms: + +The time out value can be changed by calling the static +`TestRunner::setTimeout()` method. Here is an example that sets the timeout to 1 +second instead: ``` -unsigned short ushortValue = 5; -assertEqual((unsigned short) 5, ushortValue); +void setup() { + ... + TestRunner::setTimeout(1000); + ... +} ``` -The integer type promotion rules and function overload matching rules can be -difficult to remember (and sometimes difficult to understand). The best way to -avoid these compiler errors is to make sure that the assertion parameter types -are identical, potentially using explicit casting. +A timeout value of `0` means an infinite timeout, which means that the +`testing()` test case may run forever. + +***ArduinoUnit Compatibility***: _Only available in AUnit._ ## Benchmarks @@ -551,16 +612,16 @@ microcontrollers: ``` Platform (resource) | Max | ArduinoUnit | AUnit | ---------------------------+---------+-------------+-------------| -Arduino Nano (flash) | 30720 | 54038 | 18418 | -Arduino Nano (static) | 2048 | 1061 | 908 | +Arduino Nano (flash) | 30720 | 54038 | 18666 | +Arduino Nano (static) | 2048 | 1061 | 918 | ---------------------------+---------+-------------+-------------| -Teensy LC (flash) | 63488 | 36196 | 25096 | -Teensy LC (static) | 8192 | 2980 | 2768 | +Teensy LC (flash) | 63488 | 36196 | 25228 | +Teensy LC (static) | 8192 | 2980 | 2780 | ---------------------------+---------+-------------+-------------| -Teensy 3.2 (flash) | 262144 | 51236 | 36136 | -Teensy 3.2 (static) | 65536 | 5328 | 5224 | +Teensy 3.2 (flash) | 262144 | 51236 | 36300 | +Teensy 3.2 (static) | 65536 | 5328 | 5236 | ---------------------------+---------+-------------+-------------| -ESP8266 - ESP-12E (flash) | 1044464 | does not | 267359 | +ESP8266 - ESP-12E (flash) | 1044464 | does not | 267479 | ESP8266 - ESP-12E (static) | 81920 | compile | 34564 | ---------------------------+---------+-------------+-------------| ESP8266 - ESP-01 (flash) | 499696 | does not | 267359 | From f0de6feb7ced16d8605935b52f7371c1424e1580 Mon Sep 17 00:00:00 2001 From: Brian Park Date: Fri, 16 Mar 2018 16:41:44 -0700 Subject: [PATCH 3/4] Increment to v0.2.0 since TestRunner::setTimeout() is an API addition --- CHANGELOG.md | 3 +++ library.properties | 2 +- src/AUnit.h | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eabdb5..b5ae137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +* v0.2.0 (2018-03-16) + * TestRunner can time out long running tests. Default time out is 10 + seconds, but is configurable using TestRunner::setTimeout(). * v0.1.1 (2018-03-15) * Fix small bug with Test::setPassOrFail() which caused assertXxx() macros which returned true to terminate the testing() test cases. diff --git a/library.properties b/library.properties index 470c580..db8a754 100644 --- a/library.properties +++ b/library.properties @@ -1,5 +1,5 @@ name=AUnit -version=0.1.1 +version=0.2.0 author=Brian T. Park maintainer=Brian T. Park sentence=A unit testing framework for Arduino platforms inspired by ArduinoUnit. diff --git a/src/AUnit.h b/src/AUnit.h index 64fa3fc..3424184 100644 --- a/src/AUnit.h +++ b/src/AUnit.h @@ -33,6 +33,6 @@ SOFTWARE. #include "aunit/Assertion.h" // Version format: 010203 == "1.2.3" -#define AUNIT_VERSION 000101 +#define AUNIT_VERSION 000200 #endif From 0f570ebab30793004805b47030f0fa5607d135c7 Mon Sep 17 00:00:00 2001 From: Brian Park Date: Fri, 16 Mar 2018 16:43:40 -0700 Subject: [PATCH 4/4] README.md: add externTest() and externTesting() to the list of missing ArduinoUnit macros --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c7ad091..cbe6137 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,12 @@ The following methods from ArduinoUnit are not implemented: * `assertTestSkip(name)` * `assertTestNotSkip(name)` +The following macros are not implemented because they are only needed +by the Meta Assertions: + +* `externTest()` +* `externTesting()` + ***ArduinoUnit Compatibility***: _Not implemented in AUnit._ ### Status Indicator Methods