✨ Refactor out mocha/chai (#12)

* Remove loaded check

* 🐛 Fix arduino staging

* Remove mocha and chai

* Add parallel execution

* Update reporter

* Restore archiver

* Update colour output

* Fix parallel + dependence schedule

* Fix scenario reporting

* Add deep equal

* Fix reporter

* Fix parallel

* Fix forest

* Fix dependence schedule

* Fix parallel schedule (hybrid)

* Move testbed shutdown to end

* Add recovery code

* Add errors to reporter

* Fix skipped + scheduling

* Skip unnecessary reupload

* Change parallel

package.json

@@ -22,12 +22,14 @@
     "build:cjs": "tsc --project tsconfig.build.json --module commonjs --outDir dist/cjs && npx convert-extension cjs dist/cjs/ && rm -f dist/cjs/*.map",
     "watch": "tsc -watch -p ./",
     "lint": "eslint src --ext ts",
-    "test": "npx latch test/test.ts"
+    "test": "npx ts-node test/test.ts",
+    "debugtest": "npx ts-node test/debugger.test.ts",
+    "spectest": "npx ts-node test/spec.ts"
   },
   "dependencies": {
-    "chai": "^4.3.6",
+    "ansi-colors": "^4.1.3",
     "ieee754": "^1.2.1",
-    "mocha": "10.1.0",
+    "ora": "^8.0.1",
     "source-map": "^0.7.4",
     "ts-node": "^10.5.0"
   },

src/framework/Framework.ts

@@ -1,8 +1,9 @@
-import {Testee} from './Testee';
+import {Testee, timeout} from './Testee';
 import {HybridScheduler, Scheduler} from './Scheduler';
 import {TestScenario} from './scenario/TestScenario';
 
 import {TestbedSpecification} from '../testbeds/TestbedSpecification';
+import {Reporter, SuiteResults} from './Reporter';
 
 export interface Suite {
 
@@ -30,12 +31,15 @@ export class Suite {
     public scenarios: TestScenario[] = [];
     public testees: Testee[] = [];
 
-    public constructor(title: string) {
+    public scheduler: Scheduler;
+
+    public constructor(title: string, scheduler: Scheduler = new HybridScheduler()) {
         this.title = title;
+        this.scheduler = scheduler;
     }
 
-    public testee(name: string, specification: TestbedSpecification, scheduler: Scheduler = new HybridScheduler(), options: TesteeOptions = {}) {
-        const testee = new Testee(name, specification, scheduler, options.timeout ?? 2000, options.connectionTimout ?? 5000);
+    public testee(name: string, specification: TestbedSpecification, options: TesteeOptions = {}) {
+        const testee = new Testee(name, specification, options.timeout ?? 2000, options.connectionTimout ?? 5000);
         if (options.disabled) {
             testee.skipall();
         }
@@ -61,11 +65,13 @@ export class Framework {
 
     private scheduled: Suite[] = [];
 
+    public readonly reporter: Reporter = new Reporter();
+
     private constructor() {
     }
 
-    public suite(title: string): Suite {
-        return new Suite(title);
+    public suite(title: string, scheduler: Scheduler = new HybridScheduler()): Suite {
+        return new Suite(title, scheduler);
     }
 
     public suites(): Suite[] {
@@ -80,45 +86,84 @@ export class Framework {
         return this.outputStyle;
     }
 
-    public run(suites: Suite[], cores: number = 1) {   // todo remove cores
+    public async sequential(suites: Suite[]) {
+        this.scheduled.concat(suites);
+        this.reporter.general();
+        const t0 = performance.now();
+        for (const suite of suites) {
+            for (const testee of suite.testees) {
+                const order: TestScenario[] = suite.scheduler.sequential(suite);
+                const result: SuiteResults = new SuiteResults(suite, testee);
+
+                const first: TestScenario = order[0];
+                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
+
+                await this.runSuite(result, testee, order);
+                this.reporter.report(result);
+            }
+        }
+        const t1 = performance.now();
+        this.reporter.results(t1 - t0);
+    }
+
+    public async run(suites: Suite[]) {
         this.scheduled.concat(suites);
-        suites.forEach((suite: Suite) => {
-            suite.testees.forEach((testee: Testee) => {
-                const order: TestScenario[] = testee.scheduler.schedule(suite);
+        this.reporter.general();
+        const t0 = performance.now();
+        await Promise.all(suites.map(async (suite: Suite) => {
+            await Promise.all(suite.testees.map(async (testee: Testee) => {
+                const order: TestScenario[] = suite.scheduler.sequential(suite);
+                const result: SuiteResults = new SuiteResults(suite, testee);
+
                 const first: TestScenario = order[0];
-                before('Initialize testbed', async function () {
-                    this.timeout(testee.connector.timeout);
-                    await testee.initialize(first.program, first.args ?? []).catch((e) => Promise.reject(e));
-                });
-
-                describe(`${testee.name}: ${suite.title}`, () => {
-                    // todo add parallelism
-
-                    // if (!bed.disabled) { // TODO necessary? isn't this done in de test itself?
-                    //
-                    //     after('Shutdown debugger', async function () {
-                    //         if (bed.describer.instance) {
-                    //             await bed.connection.kill();
-                    //         }
-                    //     });
-                    // }
-
-                    order.forEach((test: TestScenario) => {
-                        testee.describe(test, this.runs);
-                    });
-                });
-
-                after('Shutdown testbed', async function () {
-                    await testee.shutdown();
-                });
-            });
-        });
+                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
+
+                await this.runSuite(result, testee, order);
+                this.reporter.report(result);
+            }))
+        }))
+        const t1 = performance.now();
+        this.reporter.results(t1 - t0);
+    }
+
+    public async parallel(suites: Suite[]) {
+        this.scheduled.concat(suites);
+        this.reporter.general();
+        const t0 = performance.now();
+        await Promise.all(suites.map(async (suite: Suite) => {
+            const order: TestScenario[][] = suite.scheduler.parallel(suite, suite.testees.length);
+            await Promise.all(suite.testees.map(async (testee: Testee, i: number) => {
+                // console.log(`scheduling on ${testee.name}`)
+                const result: SuiteResults = new SuiteResults(suite, testee);
+
+                const first: TestScenario = order[i][0];
+                await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
+
+                for (let j = i; j < order.length; j += suite.testees.length) {
+                    await this.runSuite(result, testee, order[j]);
+                }
+                this.reporter.report(result);
+            }))
+
+            await Promise.all(suite.testees.map(async (testee: Testee) => {
+                await timeout<Object | void>('Shutdown testbed', testee.timeout, testee.shutdown());
+            }))
+        }))
+
+        const t1 = performance.now();
+        this.reporter.results(t1 - t0);
+    }
+
+    private async runSuite(result: SuiteResults, testee: Testee, order: TestScenario[]) {
+        for (const test of order) {
+            await testee.describe(test, result, this.runs);
+        }
     }
 
     // Analyse flakiness
-    public analyse(suite: Suite[], runs: number = 3, cores: number = 1) {
+    public analyse(suite: Suite[], runs: number = 3) {
         this.runs = runs;
-        this.run(suite, cores);
+        this.run(suite);
     }
 
     public static getImplementation() {

src/framework/MochaReporter.ts

@@ -49,7 +49,6 @@ interface Result {
 
 class MochaReporter extends reporters.Base {
     private readonly framework: Framework;
-    private coreReporter: Reporter;
 
     private archiver: Archiver;
 
@@ -72,7 +71,6 @@ class MochaReporter extends reporters.Base {
         super(runner, options);
 
         this.framework = Framework.getImplementation();
-        this.coreReporter = new Reporter(this.framework);
 
         this.archiver = new Archiver(`${process.env.TESTFILE?.replace('.asserts.wast', '.wast') ?? 'suite'}.${Date.now()}.log`);
         this.archiver.set('date', new Date(Date.now()).toISOString());