From 42cb0fa76ef2943062f2bcd379c95d31a1f2da8e Mon Sep 17 00:00:00 2001 From: Patric Feldmeier Date: Tue, 17 Aug 2021 18:30:28 +0200 Subject: [PATCH 1/8] Check if we have already reached a stopping condition before sorting the population in SimpleGA --- whisker-main/src/whisker/search/algorithms/SimpleGA.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/whisker-main/src/whisker/search/algorithms/SimpleGA.ts b/whisker-main/src/whisker/search/algorithms/SimpleGA.ts index d6c12aa4..03daa0ba 100644 --- a/whisker-main/src/whisker/search/algorithms/SimpleGA.ts +++ b/whisker-main/src/whisker/search/algorithms/SimpleGA.ts @@ -97,8 +97,10 @@ export class SimpleGA extends SearchAlgorithmDefault { let population = this.generateInitialPopulation(); await this.evaluatePopulation(population); - // Evaluate population - this.evaluateAndSortPopulation(population); + // Evaluate population, but before check if we have already reached our stopping condition + if(!(this._stoppingCondition.isFinished(this))) { + this.evaluateAndSortPopulation(population); + } while (!(this._stoppingCondition.isFinished(this))) { console.log(`Iteration ${this._iterations}, best fitness: ${this._bestFitness}`); @@ -106,7 +108,7 @@ export class SimpleGA extends SearchAlgorithmDefault { const nextGeneration = this.generateOffspringPopulation(population); await this.evaluatePopulation(nextGeneration); if(!(this._stoppingCondition.isFinished(this))) { - this.evaluateAndSortPopulation(nextGeneration) + this.evaluateAndSortPopulation(nextGeneration); } population = nextGeneration; this._iterations++; From e9164f14d1f60450fae87c5d3a3574092ed38c5e Mon Sep 17 00:00:00 2001 From: Patric Feldmeier Date: Wed, 18 Aug 2021 12:40:39 +0200 Subject: [PATCH 2/8] Adjust EventSelectors and make necessary changes to Events themselves --- .../src/whisker/testcase/TestExecutor.ts | 15 +++++++----- .../testcase/events/ClickSpriteEvent.ts | 4 ++-- .../testcase/events/ClickStageEvent.ts | 4 ++-- .../testcase/events/DragSpriteEvent.ts | 17 +++++++++---- .../whisker/testcase/events/KeyPressEvent.ts | 8 +++++-- .../whisker/testcase/events/MouseDownEvent.ts | 4 ++-- .../whisker/testcase/events/MouseMoveEvent.ts | 14 +++++++++-- .../testcase/events/MouseMoveToEvent.ts | 4 ++-- .../whisker/testcase/events/ParameterTypes.ts | 2 ++ .../whisker/testcase/events/ScratchEvent.ts | 24 +++++++++---------- .../src/whisker/testcase/events/SoundEvent.ts | 4 ++-- .../whisker/testcase/events/TypeTextEvent.ts | 4 ++-- .../src/whisker/testcase/events/WaitEvent.ts | 8 +++++-- whisker-main/src/whisker/utils/Randomness.ts | 4 ++-- .../whisker/whiskerNet/NetworkChromosome.ts | 2 +- .../src/whisker/whiskerNet/NetworkExecutor.ts | 2 +- .../NetworkChromosomeGenerator.ts | 2 +- ...etworkChromosomeGeneratorFullyConnected.ts | 2 +- .../whisker/testcase/EventSelector.test.ts | 4 ++-- .../test/whisker/utils/Randomness.test.ts | 2 +- 20 files changed, 81 insertions(+), 49 deletions(-) diff --git a/whisker-main/src/whisker/testcase/TestExecutor.ts b/whisker-main/src/whisker/testcase/TestExecutor.ts index e8208c30..54ad65cc 100644 --- a/whisker-main/src/whisker/testcase/TestExecutor.ts +++ b/whisker-main/src/whisker/testcase/TestExecutor.ts @@ -29,11 +29,12 @@ import {StatisticsCollector} from "../utils/StatisticsCollector"; import {EventObserver} from "./EventObserver"; import {seedScratch} from "../../util/random"; import {Randomness} from "../utils/Randomness"; -import VMWrapper = require("../../vm/vm-wrapper.js") import {ScratchEventExtractor} from "./ScratchEventExtractor"; import Runtime from "scratch-vm/src/engine/runtime"; import {EventSelector} from "./EventSelector"; import {ParameterTypes} from "./events/ParameterTypes"; +import {DynamicScratchEventExtractor} from "./DynamicScratchEventExtractor"; +import VMWrapper = require("../../vm/vm-wrapper.js"); export class TestExecutor { @@ -100,11 +101,11 @@ export class TestExecutor { // Check if we came closer to cover a specific block. This is only makes sense when using a SingleObjective // focused Algorithm like MIO. - if(testChromosome.targetFitness){ + if (testChromosome.targetFitness) { testChromosome.trace = new ExecutionTrace(this._vm.runtime.traceInfo.tracer.traces, events.clone()); testChromosome.coverage = currentCoverage; const currentFitness = testChromosome.targetFitness.getFitness(testChromosome); - if(testChromosome.targetFitness.compare(currentFitness, targetFitness) > 0){ + if (testChromosome.targetFitness.compare(currentFitness, targetFitness) > 0) { targetFitness = currentFitness; testChromosome.lastImprovedFitnessCodon = numCodon; } @@ -145,9 +146,11 @@ export class TestExecutor { const nextEvent: ScratchEvent = this._eventSelector.selectEvent(codons, numCodon, availableEvents); numCodon++; const args = TestExecutor.getArgs(nextEvent, codons, numCodon); - nextEvent.setParameter(args, ParameterTypes.CODON); + const parameterType = this._eventSelector instanceof DynamicScratchEventExtractor ? + ParameterTypes.CODON : ParameterTypes.RANDOM; + nextEvent.setParameter(args, parameterType); events.add([nextEvent, args]); - numCodon += nextEvent.getNumVariableParameters(); + numCodon += nextEvent.numSearchParameter(); this.notify(nextEvent, args); // Send the chosen Event including its parameters to the VM await nextEvent.apply(); @@ -168,7 +171,7 @@ export class TestExecutor { */ private static getArgs(event: ScratchEvent, codons: List, codonPosition: number): number[] { const args = []; - for (let i = 0; i < event.getNumVariableParameters(); i++) { + for (let i = 0; i < event.numSearchParameter(); i++) { args.push(codons.get(codonPosition++ % codons.size())); } return args; diff --git a/whisker-main/src/whisker/testcase/events/ClickSpriteEvent.ts b/whisker-main/src/whisker/testcase/events/ClickSpriteEvent.ts index e3136f76..d8744727 100644 --- a/whisker-main/src/whisker/testcase/events/ClickSpriteEvent.ts +++ b/whisker-main/src/whisker/testcase/events/ClickSpriteEvent.ts @@ -81,7 +81,7 @@ export class ClickSpriteEvent extends ScratchEvent { } } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } @@ -93,7 +93,7 @@ export class ClickSpriteEvent extends ScratchEvent { return [this._target, this._steps]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/ClickStageEvent.ts b/whisker-main/src/whisker/testcase/events/ClickStageEvent.ts index 76cd0a91..cf9a7f53 100644 --- a/whisker-main/src/whisker/testcase/events/ClickStageEvent.ts +++ b/whisker-main/src/whisker/testcase/events/ClickStageEvent.ts @@ -37,7 +37,7 @@ export class ClickStageEvent extends ScratchEvent { return "ClickStage" } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } @@ -49,7 +49,7 @@ export class ClickStageEvent extends ScratchEvent { return []; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts index 12d215b6..1037e97e 100644 --- a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts +++ b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts @@ -21,6 +21,8 @@ import {ScratchEvent} from "./ScratchEvent"; import {RenderedTarget} from 'scratch-vm/src/sprites/rendered-target'; import {Container} from "../../utils/Container"; +import {ParameterTypes} from "./ParameterTypes"; +import {Randomness} from "../../utils/Randomness"; export class DragSpriteEvent extends ScratchEvent { @@ -55,8 +57,15 @@ export class DragSpriteEvent extends ScratchEvent { return [this._x, this._y, this.angle, this._target.sprite.name]; } - setParameter(args: number[]): void { - this.angle = args[0]; + setParameter(args: number[], argType: ParameterTypes): void { + switch (argType) { + case ParameterTypes.RANDOM: + this.angle = Randomness.getInstance().nextInt(0, 421); + break; + case ParameterTypes.CODON: + this.angle = args[0]; + break; + } // We only disturb the target point if we have an angle smaller than 360 degrees. if (this.angle < 360) { @@ -78,11 +87,11 @@ export class DragSpriteEvent extends ScratchEvent { } } - getNumVariableParameters(): number { + numSearchParameter(): number { return 1; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts b/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts index 3790f024..b1929329 100644 --- a/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts +++ b/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts @@ -23,6 +23,7 @@ import {Container} from "../../utils/Container"; import {WaitEvent} from "./WaitEvent"; import {ParameterTypes} from "./ParameterTypes"; import {NeuroevolutionUtil} from "../../whiskerNet/NeuroevolutionUtil"; +import {Randomness} from "../../utils/Randomness"; export class KeyPressEvent extends ScratchEvent { @@ -61,7 +62,7 @@ new WaitEvent(this._steps).toJavaScript() return "KeyPress " + this._keyOption + ": " + this._steps; } - getNumVariableParameters(): number { + numSearchParameter(): number { return 1; } @@ -69,12 +70,15 @@ new WaitEvent(this._steps).toJavaScript() return [this._keyOption, this._steps]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return ["Steps"]; } setParameter(args:number[], testExecutor:ParameterTypes): void { switch (testExecutor){ + case ParameterTypes.RANDOM: + this._steps = Randomness.getInstance().nextInt(1, 421); + break; case ParameterTypes.CODON: this._steps = args[0]; break; diff --git a/whisker-main/src/whisker/testcase/events/MouseDownEvent.ts b/whisker-main/src/whisker/testcase/events/MouseDownEvent.ts index d8bc7c2b..660a870c 100644 --- a/whisker-main/src/whisker/testcase/events/MouseDownEvent.ts +++ b/whisker-main/src/whisker/testcase/events/MouseDownEvent.ts @@ -48,7 +48,7 @@ export class MouseDownEvent extends ScratchEvent { return "MouseDown " + this._value; } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } @@ -57,7 +57,7 @@ export class MouseDownEvent extends ScratchEvent { return [this._value ? 1 : 0]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts b/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts index 973951f1..530f676a 100644 --- a/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts +++ b/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts @@ -21,6 +21,7 @@ import {ScratchEvent} from "./ScratchEvent"; import {Container} from "../../utils/Container"; import {ParameterTypes} from "./ParameterTypes"; +import {Randomness} from "../../utils/Randomness"; export class MouseMoveEvent extends ScratchEvent { @@ -53,7 +54,7 @@ export class MouseMoveEvent extends ScratchEvent { return "MouseMove " + Math.trunc(this._x) + "/" + Math.trunc(this._y); } - getNumVariableParameters(): number { + numSearchParameter(): number { return 2; // x and y? } @@ -61,12 +62,21 @@ export class MouseMoveEvent extends ScratchEvent { return [this._x, this._y]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return ["X", "Y"] } setParameter(args: number[], argType: ParameterTypes): void { switch (argType) { + case ParameterTypes.RANDOM: { + const random = Randomness.getInstance(); + const randomX = random.nextInt(0, 421); + const randomY = random.nextInt(0, 361); + const fittedCoordinates = this.fitCoordinates(randomX, randomY); + this._x = fittedCoordinates.x; + this._y = fittedCoordinates.y; + break; + } case ParameterTypes.CODON: { const fittedCoordinates = this.fitCoordinates(args[0], args[1]) this._x = fittedCoordinates.x; diff --git a/whisker-main/src/whisker/testcase/events/MouseMoveToEvent.ts b/whisker-main/src/whisker/testcase/events/MouseMoveToEvent.ts index 17770c29..ff87c308 100644 --- a/whisker-main/src/whisker/testcase/events/MouseMoveToEvent.ts +++ b/whisker-main/src/whisker/testcase/events/MouseMoveToEvent.ts @@ -53,7 +53,7 @@ export class MouseMoveToEvent extends ScratchEvent { return "MouseMove " + Math.trunc(this.x) + "/" + Math.trunc(this.y); } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } @@ -61,7 +61,7 @@ export class MouseMoveToEvent extends ScratchEvent { return [this.x, this.y]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/ParameterTypes.ts b/whisker-main/src/whisker/testcase/events/ParameterTypes.ts index e021111f..b8a64369 100644 --- a/whisker-main/src/whisker/testcase/events/ParameterTypes.ts +++ b/whisker-main/src/whisker/testcase/events/ParameterTypes.ts @@ -1,4 +1,6 @@ export enum ParameterTypes { + "RANDOM", + "CODON", "REGRESSION" diff --git a/whisker-main/src/whisker/testcase/events/ScratchEvent.ts b/whisker-main/src/whisker/testcase/events/ScratchEvent.ts index 3e1ac8da..a07f99c7 100644 --- a/whisker-main/src/whisker/testcase/events/ScratchEvent.ts +++ b/whisker-main/src/whisker/testcase/events/ScratchEvent.ts @@ -31,27 +31,27 @@ export abstract class ScratchEvent { abstract apply(): Promise; /** - * Returns the number of variable parameters required by this event. + * Returns the number of parameters that will be defined during search. */ - abstract getNumVariableParameters(): number; + abstract numSearchParameter(): number; + + /** + * Returns the name(s) of parameter(s) defined during search. + */ + abstract getSearchParameterNames(): string[]; /** * Sets the parameter(s) of this event using the given arguments. * @param args the values to which the parameters of this event should be set to - * @param argType the type of the given arguments decides how they should be interpreted as parameters. + * @param argType the type of the given arguments decide how they should be interpreted as parameters. */ abstract setParameter(args: number[], argType: ParameterTypes): void; /** - * Returns the parameter(s) of this event. + * Returns all parameter(s) of this event. */ abstract getParameter(): (number | string | RenderedTarget) []; - /** - * Returns the name(s) of variable parameter(s). - */ - abstract getVariableParameterNames(): string[]; - /** * Transforms the event into an executable Whisker-Test statement. */ @@ -63,9 +63,9 @@ export abstract class ScratchEvent { abstract toString(): string; /** - * Returns an identifier as string. Events containing variable parameters obtain the same identifier and - * Events whose parameters are determined by the ScratchEventExtractor get different identifiers. - * The id is used to query the right RegressionNode if variable parameters for a specific Event are needed. + * Returns an identifier as string. Events containing parameters defined during search obtain the same + * identifier and Events whose parameters are determined by the ScratchEventExtractor get different identifiers. + * The id is used to query the right RegressionNode if search defined parameters for a specific Event are needed. */ abstract stringIdentifier():string; diff --git a/whisker-main/src/whisker/testcase/events/SoundEvent.ts b/whisker-main/src/whisker/testcase/events/SoundEvent.ts index 2c728b2f..f7397338 100644 --- a/whisker-main/src/whisker/testcase/events/SoundEvent.ts +++ b/whisker-main/src/whisker/testcase/events/SoundEvent.ts @@ -37,7 +37,7 @@ export class SoundEvent extends ScratchEvent { throw new NotYetImplementedException(); } - getNumVariableParameters(): number { + numSearchParameter(): number { return 1; } @@ -45,7 +45,7 @@ export class SoundEvent extends ScratchEvent { return [this._volume]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return ["Volume"]; } diff --git a/whisker-main/src/whisker/testcase/events/TypeTextEvent.ts b/whisker-main/src/whisker/testcase/events/TypeTextEvent.ts index f6ded097..6feae01d 100644 --- a/whisker-main/src/whisker/testcase/events/TypeTextEvent.ts +++ b/whisker-main/src/whisker/testcase/events/TypeTextEvent.ts @@ -48,7 +48,7 @@ export class TypeTextEvent extends ScratchEvent { return `TypeText '${this._text}'` } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; // Text } @@ -56,7 +56,7 @@ export class TypeTextEvent extends ScratchEvent { return [this._text]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/WaitEvent.ts b/whisker-main/src/whisker/testcase/events/WaitEvent.ts index 40642bc6..802f26cc 100644 --- a/whisker-main/src/whisker/testcase/events/WaitEvent.ts +++ b/whisker-main/src/whisker/testcase/events/WaitEvent.ts @@ -22,6 +22,7 @@ import {ScratchEvent} from "./ScratchEvent"; import {Container} from "../../utils/Container"; import {ParameterTypes} from "./ParameterTypes"; import {NeuroevolutionUtil} from "../../whiskerNet/NeuroevolutionUtil"; +import {Randomness} from "../../utils/Randomness"; export class WaitEvent extends ScratchEvent { @@ -44,7 +45,7 @@ export class WaitEvent extends ScratchEvent { return "Wait for " + this.steps + " steps"; } - getNumVariableParameters(): number { + numSearchParameter(): number { return 1; } @@ -52,12 +53,15 @@ export class WaitEvent extends ScratchEvent { return [this.steps]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return ["Duration"]; } setParameter(args:number[], testExecutor:ParameterTypes): void { switch (testExecutor){ + case ParameterTypes.RANDOM: + this.steps = Randomness.getInstance().nextInt(0,421); + break; case ParameterTypes.CODON: this.steps = args[0]; break; diff --git a/whisker-main/src/whisker/utils/Randomness.ts b/whisker-main/src/whisker/utils/Randomness.ts index bc287dc6..bf2f9a53 100644 --- a/whisker-main/src/whisker/utils/Randomness.ts +++ b/whisker-main/src/whisker/utils/Randomness.ts @@ -84,8 +84,8 @@ export class Randomness { /** * Pick a random integer from a range - * @param min Lower bound of range - * @param max Upper bound of range + * @param min Lower bound of range, included + * @param max Upper bound of range, excluded */ public nextInt(min: number, max: number): number { return Math.floor(this.next(min, max)); diff --git a/whisker-main/src/whisker/whiskerNet/NetworkChromosome.ts b/whisker-main/src/whisker/whiskerNet/NetworkChromosome.ts index 6f23581d..0ab2e3d4 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkChromosome.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkChromosome.ts @@ -289,7 +289,7 @@ export class NetworkChromosome extends Chromosome { const classificationNode = new ClassificationNode(this.allNodes.size(), event, ActivationFunction.SIGMOID); this.allNodes.add(classificationNode); this.connectOutputNode(classificationNode); - for (const parameter of event.getVariableParameterNames()) { + for (const parameter of event.getSearchParameterNames()) { const regressionNode = new RegressionNode(this.allNodes.size(), event, parameter, ActivationFunction.NONE) this.allNodes.add(regressionNode); this.connectOutputNode(regressionNode); diff --git a/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts b/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts index 74374bf1..4a311d1b 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts @@ -148,7 +148,7 @@ export class NetworkExecutor { // Select the nextEvent, set its parameters and send it to the Scratch-VM const nextEvent: ScratchEvent = this.availableEvents.get(indexOfMaxValue); let args = []; - if (nextEvent.getNumVariableParameters() > 0) { + if (nextEvent.numSearchParameter() > 0) { args = NetworkExecutor.getArgs(nextEvent, network); nextEvent.setParameter(args, ParameterTypes.REGRESSION); } diff --git a/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGenerator.ts b/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGenerator.ts index f06b2f23..162b729f 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGenerator.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGenerator.ts @@ -59,7 +59,7 @@ export abstract class NetworkChromosomeGenerator implements ChromosomeGenerator< */ protected addRegressionNodes(allNodes: List, parameterizedEvents: List, nodeId: number): void { for (const event of parameterizedEvents) { - for (const parameter of event.getVariableParameterNames()) { + for (const parameter of event.getSearchParameterNames()) { // Create the regression Node and add it to the NodeList const regressionNode = new RegressionNode(nodeId++, event, parameter, ActivationFunction.NONE) allNodes.add(regressionNode) diff --git a/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGeneratorFullyConnected.ts b/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGeneratorFullyConnected.ts index ca3c3f4e..1cb24977 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGeneratorFullyConnected.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGeneratorFullyConnected.ts @@ -71,7 +71,7 @@ export class NetworkChromosomeGeneratorFullyConnected extends NetworkChromosomeG } // Add regression nodes for each parameter of each parameterized Event - const parameterizedEvents = this._scratchEvents.filter(event => event.getNumVariableParameters() > 0); + const parameterizedEvents = this._scratchEvents.filter(event => event.numSearchParameter() > 0); if (!parameterizedEvents.isEmpty()) { this.addRegressionNodes(allNodes, parameterizedEvents, nodeId); } diff --git a/whisker-main/test/whisker/testcase/EventSelector.test.ts b/whisker-main/test/whisker/testcase/EventSelector.test.ts index 35cd1b8e..5c715343 100644 --- a/whisker-main/test/whisker/testcase/EventSelector.test.ts +++ b/whisker-main/test/whisker/testcase/EventSelector.test.ts @@ -5,10 +5,10 @@ import {RenderedTarget} from "scratch-vm/src/sprites/rendered-target"; class DummyEvent extends ScratchEvent { - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } stringIdentifier(): string { diff --git a/whisker-main/test/whisker/utils/Randomness.test.ts b/whisker-main/test/whisker/utils/Randomness.test.ts index 105478b7..0d83e53c 100644 --- a/whisker-main/test/whisker/utils/Randomness.test.ts +++ b/whisker-main/test/whisker/utils/Randomness.test.ts @@ -29,7 +29,7 @@ describe("Randomness", () => { const num = random.nextInt(0, 10); expect(num).toBeGreaterThanOrEqual(0); - expect(num).toBeLessThanOrEqual(10); + expect(num).toBeLessThan(10); }); test("Create a float in [0,1]", () => { From 0d7ebf36caf935dc1d8ded0b5b1b50d262096b5c Mon Sep 17 00:00:00 2001 From: Patric Feldmeier Date: Wed, 18 Aug 2021 15:51:03 +0200 Subject: [PATCH 3/8] EventExtractor changes to meet specified behaviour --- .../testcase/NaiveScratchEventExtractor.ts | 30 ++++-- .../whisker/testcase/ScratchEventExtractor.ts | 10 +- .../testcase/StaticScratchEventExtractor.ts | 101 +++++++++++++++++- 3 files changed, 125 insertions(+), 16 deletions(-) diff --git a/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts index 3d5f636a..39430fff 100644 --- a/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts @@ -27,37 +27,53 @@ import {ScratchEventExtractor} from "./ScratchEventExtractor"; import {MouseDownEvent} from "./events/MouseDownEvent"; import {MouseMoveEvent} from "./events/MouseMoveEvent"; import {KeyPressEvent} from "./events/KeyPressEvent"; -import {SoundEvent} from "./events/SoundEvent"; import {TypeTextEvent} from "./events/TypeTextEvent"; import {DragSpriteEvent} from "./events/DragSpriteEvent"; +import {ClickSpriteEvent} from "./events/ClickSpriteEvent"; +import {ClickStageEvent} from "./events/ClickStageEvent"; +import {Randomness} from "../utils/Randomness"; +import {SoundEvent} from "./events/SoundEvent"; export class NaiveScratchEventExtractor extends ScratchEventExtractor { + // TODO: Additional keys? private readonly KEYS = ['space', 'left arrow', 'up arrow', 'right arrow', 'down arrow', 'enter']; - private readonly _text; + private readonly _random: Randomness; + /** + * NaiveScratchEventExtractor adds every type of supported Whisker-Event to the set of avilalbe events. + * Whenever a parameter is required, it is randomly selected. + * @param vm the Scratch-VM + */ constructor(vm: VirtualMachine) { super(vm); - this._text = this._randomText(3); + this._random = Randomness.getInstance(); } public extractEvents(vm: VirtualMachine): List { const eventList = new List(); + eventList.add(new ClickStageEvent()); eventList.add(new WaitEvent()); - eventList.add(new TypeTextEvent(this._text)); - eventList.addList(this._getTypeTextEvents()); // Just one random string + eventList.add(new TypeTextEvent(this._randomText(3))); eventList.add(new MouseDownEvent(true)); eventList.add(new MouseDownEvent(false)); eventList.add(new MouseMoveEvent()); - // eventList.add(new SoundEvent()); // Not implemented yet + // eventList.add(new SoundEvent()) not implemented yet + + // Add specified keys. for (const key of this.KEYS) { eventList.add(new KeyPressEvent(key)); } + + // Add events requiring a targets as parameters. for (const target of vm.runtime.targets) { - eventList.add(new DragSpriteEvent(target)); + const x = this._random.nextInt(-240, 241); + const y = this._random.nextInt(-180, 181); + eventList.add(new DragSpriteEvent(target, x, y)); + eventList.add(new ClickSpriteEvent(target)); } return eventList.distinctObjects(); } diff --git a/whisker-main/src/whisker/testcase/ScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/ScratchEventExtractor.ts index e6d0c003..a806ecee 100644 --- a/whisker-main/src/whisker/testcase/ScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/ScratchEventExtractor.ts @@ -101,8 +101,6 @@ export abstract class ScratchEventExtractor { } } - - // TODO: How to handle event parameters? protected _extractEventsFromBlock(target, block): List { const eventList = new List(); if (typeof block.opcode === 'undefined') { @@ -123,7 +121,8 @@ export abstract class ScratchEventExtractor { break; } case 'sensing_mousex': - case 'sensing_mousey': { + case 'sensing_mousey': + case 'pen_penDown': { // Mouse move eventList.add(new MouseMoveEvent()); break; @@ -190,7 +189,6 @@ export abstract class ScratchEventExtractor { const field = target.blocks.getFields(distanceMenuBlock); const value = field.DISTANCETOMENU.value; if (value == "_mouse_") { - // TODO: Maybe could determine position to move to here? eventList.add(new MouseMoveEvent()); } break; @@ -207,10 +205,6 @@ export abstract class ScratchEventExtractor { eventList.add(new MouseDownEvent(!isMouseDown)); break; } - case 'pen_penDown': { - eventList.add(new MouseMoveEvent()) - break; - } case 'sensing_askandwait': // Type text if (Container.vmWrapper.isQuestionAsked()) { diff --git a/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts index b6aaad7b..ba71af3a 100644 --- a/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts @@ -24,11 +24,31 @@ import VirtualMachine from 'scratch-vm/src/virtual-machine.js'; import {ScratchEvent} from "./events/ScratchEvent"; import {WaitEvent} from "./events/WaitEvent"; import {ScratchEventExtractor} from "./ScratchEventExtractor"; +import {KeyPressEvent} from "./events/KeyPressEvent"; +import {MouseMoveEvent} from "./events/MouseMoveEvent"; +import {Randomness} from "../utils/Randomness"; +import {DragSpriteEvent} from "./events/DragSpriteEvent"; +import {MouseDownEvent} from "./events/MouseDownEvent"; +import {ClickSpriteEvent} from "./events/ClickSpriteEvent"; +import {ClickStageEvent} from "./events/ClickStageEvent"; +import {SoundEvent} from "./events/SoundEvent"; +import {TypeTextEvent} from "./events/TypeTextEvent"; export class StaticScratchEventExtractor extends ScratchEventExtractor { - constructor (vm: VirtualMachine) { +// TODO: Additional keys? + private readonly KEYS = ['space', 'left arrow', 'up arrow', 'right arrow', 'down arrow', 'enter']; + + private readonly _random: Randomness; + + /** + * StaticScratchEventExtractor only adds event for which corresponding event handler exist. However, as opposed + * to the DynamicScratchEventExtractor, the parameters are chosen randomly and not inferred if possible from the VM. + * @param vm the Scratch-VM + */ + constructor(vm: VirtualMachine) { super(vm); + this._random = Randomness.getInstance(); } public extractEvents(vm: VirtualMachine): List { @@ -48,4 +68,83 @@ export class StaticScratchEventExtractor extends ScratchEventExtractor { return eventList.distinctObjects(); } + + /** + * Extract events for which corresponding event handler exist. The parameter of the selected events are chosen + * randomly. + * @param target the sprite wrapped in a RenderedTarget + * @param block the given block from which events might get extracted. + * @returns List containing all extracted events. + */ + protected _extractEventsFromBlock(target, block): List { + const eventList = new List(); + if (typeof block.opcode === 'undefined') { + return eventList; + } + + switch (target.blocks.getOpcode(block)) { + case 'event_whenkeypressed': + case 'sensing_keypressed': { + // Only add if we have not yet found any keyPress-Events. No need to add all keys several times + if (!eventList.getElements().some(event => event instanceof KeyPressEvent)) { + for (const key of this.KEYS) { + eventList.add(new KeyPressEvent(key)); + } + } + break; + } + case 'sensing_mousex': + case 'sensing_mousey': + case 'pen_penDown': { + // Mouse move + eventList.add(new MouseMoveEvent()); + break; + } + case 'sensing_touchingobject': + case 'sensing_touchingcolor' : { + const x = this._random.nextInt(-240, 241); + const y = this._random.nextInt(-180, 181); + eventList.add(new DragSpriteEvent(target, x, y)); + break; + } + case 'sensing_distanceto': { + const distanceMenuBlock = target.blocks.getBlock(block.inputs.DISTANCETOMENU.block); + const field = target.blocks.getFields(distanceMenuBlock); + const value = field.DISTANCETOMENU.value; + if (value == "_mouse_") { + eventList.add(new MouseMoveEvent()); + } + break; + } + case 'motion_pointtowards': { + const towards = target.blocks.getBlock(block.inputs.TOWARDS.block) + if (towards.fields.TOWARDS.value === '_mouse_') + eventList.add(new MouseMoveEvent()); + break; + } + case 'sensing_mousedown': { + // Mouse down + eventList.add(new MouseDownEvent(true)); + eventList.add(new MouseDownEvent(false)); + break; + } + case 'sensing_askandwait': + // Type text + eventList.add(new TypeTextEvent(this._randomText(3))); + break; + case 'event_whenthisspriteclicked': + // Click sprite + eventList.add(new ClickSpriteEvent(target)); + break; + case 'event_whenstageclicked': + // Click stage + eventList.add(new ClickStageEvent()); + break; + case 'event_whengreaterthan': + // Sound + eventList.add(new SoundEvent()); + break; + } + return eventList; + } } From 0ebf5f755480f8023025e33298deee484ae38a10 Mon Sep 17 00:00:00 2001 From: Patric Feldmeier Date: Wed, 18 Aug 2021 19:24:41 +0200 Subject: [PATCH 4/8] Define Codons as ParameterType when we actually have codons --- .../src/whisker/testcase/NaiveScratchEventExtractor.ts | 10 ++++++---- whisker-main/src/whisker/testcase/TestExecutor.ts | 7 ++----- .../src/whisker/testcase/events/DragSpriteEvent.ts | 8 ++++---- .../src/whisker/testcase/events/KeyPressEvent.ts | 10 +++++----- .../src/whisker/testcase/events/MouseMoveEvent.ts | 10 +++++----- .../events/{ParameterTypes.ts => ParameterType.ts} | 2 +- .../src/whisker/testcase/events/ScratchEvent.ts | 4 ++-- whisker-main/src/whisker/testcase/events/WaitEvent.ts | 10 +++++----- whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts | 4 ++-- 9 files changed, 32 insertions(+), 33 deletions(-) rename whisker-main/src/whisker/testcase/events/{ParameterTypes.ts => ParameterType.ts} (62%) diff --git a/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts index 39430fff..21f121e5 100644 --- a/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts @@ -70,10 +70,12 @@ export class NaiveScratchEventExtractor extends ScratchEventExtractor { // Add events requiring a targets as parameters. for (const target of vm.runtime.targets) { - const x = this._random.nextInt(-240, 241); - const y = this._random.nextInt(-180, 181); - eventList.add(new DragSpriteEvent(target, x, y)); - eventList.add(new ClickSpriteEvent(target)); + if(!target.isStage) { + const x = this._random.nextInt(-240, 241); + const y = this._random.nextInt(-180, 181); + eventList.add(new DragSpriteEvent(target, x, y)); + eventList.add(new ClickSpriteEvent(target)); + } } return eventList.distinctObjects(); } diff --git a/whisker-main/src/whisker/testcase/TestExecutor.ts b/whisker-main/src/whisker/testcase/TestExecutor.ts index 54ad65cc..d87a24e1 100644 --- a/whisker-main/src/whisker/testcase/TestExecutor.ts +++ b/whisker-main/src/whisker/testcase/TestExecutor.ts @@ -32,7 +32,7 @@ import {Randomness} from "../utils/Randomness"; import {ScratchEventExtractor} from "./ScratchEventExtractor"; import Runtime from "scratch-vm/src/engine/runtime"; import {EventSelector} from "./EventSelector"; -import {ParameterTypes} from "./events/ParameterTypes"; +import {ParameterType} from "./events/ParameterType"; import {DynamicScratchEventExtractor} from "./DynamicScratchEventExtractor"; import VMWrapper = require("../../vm/vm-wrapper.js"); @@ -82,7 +82,6 @@ export class TestExecutor { while (numCodon < codons.size() && (this._projectRunning || this.hasActionEvents(availableEvents))) { availableEvents = this._eventExtractor.extractEvents(this._vm); - if (availableEvents.isEmpty()) { console.log("Whisker-Main: No events available for project."); break; @@ -146,9 +145,7 @@ export class TestExecutor { const nextEvent: ScratchEvent = this._eventSelector.selectEvent(codons, numCodon, availableEvents); numCodon++; const args = TestExecutor.getArgs(nextEvent, codons, numCodon); - const parameterType = this._eventSelector instanceof DynamicScratchEventExtractor ? - ParameterTypes.CODON : ParameterTypes.RANDOM; - nextEvent.setParameter(args, parameterType); + nextEvent.setParameter(args, ParameterType.CODON); events.add([nextEvent, args]); numCodon += nextEvent.numSearchParameter(); this.notify(nextEvent, args); diff --git a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts index 1037e97e..1e6e9a8e 100644 --- a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts +++ b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts @@ -21,7 +21,7 @@ import {ScratchEvent} from "./ScratchEvent"; import {RenderedTarget} from 'scratch-vm/src/sprites/rendered-target'; import {Container} from "../../utils/Container"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; import {Randomness} from "../../utils/Randomness"; @@ -57,12 +57,12 @@ export class DragSpriteEvent extends ScratchEvent { return [this._x, this._y, this.angle, this._target.sprite.name]; } - setParameter(args: number[], argType: ParameterTypes): void { + setParameter(args: number[], argType: ParameterType): void { switch (argType) { - case ParameterTypes.RANDOM: + case ParameterType.RANDOM: this.angle = Randomness.getInstance().nextInt(0, 421); break; - case ParameterTypes.CODON: + case ParameterType.CODON: this.angle = args[0]; break; } diff --git a/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts b/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts index b1929329..1ab8d387 100644 --- a/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts +++ b/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts @@ -21,7 +21,7 @@ import {ScratchEvent} from "./ScratchEvent"; import {Container} from "../../utils/Container"; import {WaitEvent} from "./WaitEvent"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; import {NeuroevolutionUtil} from "../../whiskerNet/NeuroevolutionUtil"; import {Randomness} from "../../utils/Randomness"; @@ -74,15 +74,15 @@ new WaitEvent(this._steps).toJavaScript() return ["Steps"]; } - setParameter(args:number[], testExecutor:ParameterTypes): void { + setParameter(args:number[], testExecutor:ParameterType): void { switch (testExecutor){ - case ParameterTypes.RANDOM: + case ParameterType.RANDOM: this._steps = Randomness.getInstance().nextInt(1, 421); break; - case ParameterTypes.CODON: + case ParameterType.CODON: this._steps = args[0]; break; - case ParameterTypes.REGRESSION: + case ParameterType.REGRESSION: this._steps = Math.round(NeuroevolutionUtil.relu(args[0])); break; } diff --git a/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts b/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts index 530f676a..0841c493 100644 --- a/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts +++ b/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts @@ -20,7 +20,7 @@ import {ScratchEvent} from "./ScratchEvent"; import {Container} from "../../utils/Container"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; import {Randomness} from "../../utils/Randomness"; export class MouseMoveEvent extends ScratchEvent { @@ -66,9 +66,9 @@ export class MouseMoveEvent extends ScratchEvent { return ["X", "Y"] } - setParameter(args: number[], argType: ParameterTypes): void { + setParameter(args: number[], argType: ParameterType): void { switch (argType) { - case ParameterTypes.RANDOM: { + case ParameterType.RANDOM: { const random = Randomness.getInstance(); const randomX = random.nextInt(0, 421); const randomY = random.nextInt(0, 361); @@ -77,13 +77,13 @@ export class MouseMoveEvent extends ScratchEvent { this._y = fittedCoordinates.y; break; } - case ParameterTypes.CODON: { + case ParameterType.CODON: { const fittedCoordinates = this.fitCoordinates(args[0], args[1]) this._x = fittedCoordinates.x; this._y = fittedCoordinates.y; break; } - case ParameterTypes.REGRESSION: { + case ParameterType.REGRESSION: { this._x = Math.tanh(args[0]) * 240; this._y = Math.tanh(args[1]) * 180; break; diff --git a/whisker-main/src/whisker/testcase/events/ParameterTypes.ts b/whisker-main/src/whisker/testcase/events/ParameterType.ts similarity index 62% rename from whisker-main/src/whisker/testcase/events/ParameterTypes.ts rename to whisker-main/src/whisker/testcase/events/ParameterType.ts index b8a64369..7b9aa75e 100644 --- a/whisker-main/src/whisker/testcase/events/ParameterTypes.ts +++ b/whisker-main/src/whisker/testcase/events/ParameterType.ts @@ -1,4 +1,4 @@ -export enum ParameterTypes { +export enum ParameterType { "RANDOM", "CODON", diff --git a/whisker-main/src/whisker/testcase/events/ScratchEvent.ts b/whisker-main/src/whisker/testcase/events/ScratchEvent.ts index a07f99c7..05c3cc64 100644 --- a/whisker-main/src/whisker/testcase/events/ScratchEvent.ts +++ b/whisker-main/src/whisker/testcase/events/ScratchEvent.ts @@ -20,7 +20,7 @@ import {RenderedTarget} from 'scratch-vm/src/sprites/rendered-target'; import {Container} from "../../utils/Container"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; export abstract class ScratchEvent { @@ -45,7 +45,7 @@ export abstract class ScratchEvent { * @param args the values to which the parameters of this event should be set to * @param argType the type of the given arguments decide how they should be interpreted as parameters. */ - abstract setParameter(args: number[], argType: ParameterTypes): void; + abstract setParameter(args: number[], argType: ParameterType): void; /** * Returns all parameter(s) of this event. diff --git a/whisker-main/src/whisker/testcase/events/WaitEvent.ts b/whisker-main/src/whisker/testcase/events/WaitEvent.ts index 802f26cc..0f05b3be 100644 --- a/whisker-main/src/whisker/testcase/events/WaitEvent.ts +++ b/whisker-main/src/whisker/testcase/events/WaitEvent.ts @@ -20,7 +20,7 @@ import {ScratchEvent} from "./ScratchEvent"; import {Container} from "../../utils/Container"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; import {NeuroevolutionUtil} from "../../whiskerNet/NeuroevolutionUtil"; import {Randomness} from "../../utils/Randomness"; @@ -57,15 +57,15 @@ export class WaitEvent extends ScratchEvent { return ["Duration"]; } - setParameter(args:number[], testExecutor:ParameterTypes): void { + setParameter(args:number[], testExecutor:ParameterType): void { switch (testExecutor){ - case ParameterTypes.RANDOM: + case ParameterType.RANDOM: this.steps = Randomness.getInstance().nextInt(0,421); break; - case ParameterTypes.CODON: + case ParameterType.CODON: this.steps = args[0]; break; - case ParameterTypes.REGRESSION: + case ParameterType.REGRESSION: this.steps = Math.round(NeuroevolutionUtil.relu(args[0])); break; } diff --git a/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts b/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts index 4a311d1b..bd505c60 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts @@ -13,7 +13,7 @@ import {InputExtraction} from "./InputExtraction"; import {NeuroevolutionUtil} from "./NeuroevolutionUtil"; import {ScratchEventExtractor} from "../testcase/ScratchEventExtractor"; import {StaticScratchEventExtractor} from "../testcase/StaticScratchEventExtractor"; -import {ParameterTypes} from "../testcase/events/ParameterTypes"; +import {ParameterType} from "../testcase/events/ParameterType"; import Runtime from "scratch-vm/src/engine/runtime" import {WhiskerSearchConfiguration} from "../utils/WhiskerSearchConfiguration"; import {NeuroevolutionScratchEventExtractor} from "../testcase/NeuroevolutionScratchEventExtractor"; @@ -150,7 +150,7 @@ export class NetworkExecutor { let args = []; if (nextEvent.numSearchParameter() > 0) { args = NetworkExecutor.getArgs(nextEvent, network); - nextEvent.setParameter(args, ParameterTypes.REGRESSION); + nextEvent.setParameter(args, ParameterType.REGRESSION); } events.add([nextEvent, args]); this.notify(nextEvent, args); From 771f8c94b7f831130d9ca60d358d76df34304494 Mon Sep 17 00:00:00 2001 From: Patric Feldmeier Date: Thu, 19 Aug 2021 11:05:34 +0200 Subject: [PATCH 5/8] Remodel RandomTestGenerator to generate a sequence of randomly chosen events in each Iteration --- config/random.json | 42 +++++++++ .../testcase/StaticScratchEventExtractor.ts | 5 +- .../src/whisker/testcase/TestExecutor.ts | 58 ++++++++++++- .../testgenerator/RandomTestGenerator.ts | 85 ++++++++++++++----- .../utils/WhiskerSearchConfiguration.ts | 3 +- 5 files changed, 167 insertions(+), 26 deletions(-) create mode 100644 config/random.json diff --git a/config/random.json b/config/random.json new file mode 100644 index 00000000..8b41b1a7 --- /dev/null +++ b/config/random.json @@ -0,0 +1,42 @@ +{ + "test-generator": "random", + "algorithm": "random", + "minEventSize": 2, + "maxEventSize": 30, + "extractor": "static", + "fitness-function": { + "type": "statement", + "targets": [] + }, + "stopping-condition": { + "type": "one-of", + "conditions": [ + { + "type": "fixed-time", + "duration": 6000000 + }, + { + "type": "optimal" + } + ] + }, + "crossover": { + "operator": "singlepointrelative", + "probability": 1 + }, + "mutation": { + "operator": "biasedVariablelengthConstrained", + "probability": 1, + "gaussianMutationPower": 5, + "maxMutationCountStart": 0, + "maxMutationCountFocusedPhase": 10 + }, + "selection": { + "operator": "rank", + "randomSelectionProbabilityStart": 0.5, + "randomSelectionProbabilityFocusedPhase": 0 + }, + "waitStepUpperBound": 200, + "click-duration": 3, + "press-duration": 50 +} diff --git a/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts index ba71af3a..693f248e 100644 --- a/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts @@ -36,14 +36,15 @@ import {TypeTextEvent} from "./events/TypeTextEvent"; export class StaticScratchEventExtractor extends ScratchEventExtractor { -// TODO: Additional keys? + // TODO: Additional keys? private readonly KEYS = ['space', 'left arrow', 'up arrow', 'right arrow', 'down arrow', 'enter']; private readonly _random: Randomness; /** * StaticScratchEventExtractor only adds event for which corresponding event handler exist. However, as opposed - * to the DynamicScratchEventExtractor, the parameters are chosen randomly and not inferred if possible from the VM. + * to the DynamicScratchEventExtractor, the parameters are chosen randomly and not inferred from the VM whenever + * possible. * @param vm the Scratch-VM */ constructor(vm: VirtualMachine) { diff --git a/whisker-main/src/whisker/testcase/TestExecutor.ts b/whisker-main/src/whisker/testcase/TestExecutor.ts index d87a24e1..5df9b41b 100644 --- a/whisker-main/src/whisker/testcase/TestExecutor.ts +++ b/whisker-main/src/whisker/testcase/TestExecutor.ts @@ -33,7 +33,6 @@ import {ScratchEventExtractor} from "./ScratchEventExtractor"; import Runtime from "scratch-vm/src/engine/runtime"; import {EventSelector} from "./EventSelector"; import {ParameterType} from "./events/ParameterType"; -import {DynamicScratchEventExtractor} from "./DynamicScratchEventExtractor"; import VMWrapper = require("../../vm/vm-wrapper.js"); @@ -63,6 +62,10 @@ export class TestExecutor { } } + /** + * Executes a chromosome by selecting events according to the chromosome's defined genes. + * @param testChromosome the testChromosome that should be executed. + */ async execute(testChromosome: TestChromosome): Promise { const events = new List<[ScratchEvent, number[]]>(); @@ -131,6 +134,59 @@ export class TestExecutor { return testChromosome.trace; } + /** + * Randomly executes events selected from the available event Set. + * @param randomEventChromosome the chromosome in which the executed trace will be saved in. + * @param numberOfEvents the number of events that should be executed. + */ + async executeRandomEvents(randomEventChromosome: TestChromosome, numberOfEvents: number): Promise { + seedScratch(String(Randomness.getInitialSeed())); + const _onRunStop = this.projectStopped.bind(this); + this._vm.on(Runtime.PROJECT_RUN_STOP, _onRunStop); + this._projectRunning = true; + this._vmWrapper.start(); + let availableEvents = this._eventExtractor.extractEvents(this._vm); + let eventCount = 0; + const random = Randomness.getInstance(); + const events = new List<[ScratchEvent, number[]]>(); + + while (eventCount < numberOfEvents && (this._projectRunning || this.hasActionEvents(availableEvents))) { + availableEvents = this._eventExtractor.extractEvents(this._vm); + if (availableEvents.isEmpty()) { + console.log("Whisker-Main: No events available for project."); + break; + } + + // Randomly select an event and increase the event count. + const eventIndex = random.nextInt(0, availableEvents.size()); + randomEventChromosome.getGenes().add(eventIndex); + const event = availableEvents.get(eventIndex); + eventCount++; + + // If the selected required additional parameters; select them randomly as well. + if (event.numSearchParameter() > 0) { + // args are set in the event itself since the event knows which range of random values makes sense. + event.setParameter(null, ParameterType.RANDOM); + } + events.add([event, event.getParameter()]); + await event.apply(); + StatisticsCollector.getInstance().incrementEventsCount() + + // Send a WaitEvent to the VM + const waitEvent = new WaitEvent(1); + events.add([waitEvent, []]); + await waitEvent.apply(); + + } + const trace = new ExecutionTrace(this._vm.runtime.traceInfo.tracer.traces, events); + randomEventChromosome.coverage = this._vm.runtime.traceInfo.tracer.coverage as Set; + randomEventChromosome.trace = trace; + this._vmWrapper.end(); + this.resetState(); + StatisticsCollector.getInstance().numberFitnessEvaluations++; + return trace; + } + /** * Selects and sends the next Event to the VM * @param codons the list of codons deciding which event and parameters to take diff --git a/whisker-main/src/whisker/testgenerator/RandomTestGenerator.ts b/whisker-main/src/whisker/testgenerator/RandomTestGenerator.ts index 9c18ffaf..5220f10b 100644 --- a/whisker-main/src/whisker/testgenerator/RandomTestGenerator.ts +++ b/whisker-main/src/whisker/testgenerator/RandomTestGenerator.ts @@ -19,23 +19,21 @@ */ import {TestGenerator} from './TestGenerator'; -import {ScratchProject} from '../scratch/ScratchProject'; import {List} from '../utils/List'; -import {SearchAlgorithmProperties} from '../search/SearchAlgorithmProperties'; -import {ChromosomeGenerator} from "../search/ChromosomeGenerator"; import {TestChromosome} from "../testcase/TestChromosome"; import {SearchAlgorithm} from "../search/SearchAlgorithm"; -import {Selection} from "../search/Selection"; import {NotSupportedFunctionException} from "../core/exceptions/NotSupportedFunctionException"; import {FitnessFunction} from "../search/FitnessFunction"; import {StatisticsCollector} from "../utils/StatisticsCollector"; import {WhiskerTestListWithSummary} from "./WhiskerTestListWithSummary"; -import {LocalSearch} from "../search/operators/LocalSearch/LocalSearch"; +import {Randomness} from "../utils/Randomness"; +import {Container} from "../utils/Container"; +import {TestExecutor} from "../testcase/TestExecutor"; +import {WhiskerSearchConfiguration} from "../utils/WhiskerSearchConfiguration"; /** - * A naive approach to generating tests is to simply - * use the chromosome factory and generate completely - * random tests. + * A naive approach to generating tests by always selecting a random event from the set of available events + * determined by the ScratchEventSelector. */ export class RandomTestGenerator extends TestGenerator implements SearchAlgorithm { @@ -47,7 +45,7 @@ export class RandomTestGenerator extends TestGenerator implements SearchAlgorith /** * Saves the number of Generations. */ - private _iterations = 0; + private _iterations: number; /** * Saves the best performing chromosomes seen so far. @@ -59,27 +57,53 @@ export class RandomTestGenerator extends TestGenerator implements SearchAlgorith */ private _archive = new Map(); - async generateTests(project: ScratchProject): Promise { + /** + * Boolean determining if we have reached full test coverage. + */ + protected _fullCoverageReached = false; + + /** + * The minimum number of randomly selected events. + */ + private readonly minSize: number + + /** + * The maximum number of randomly selected events. + */ + private readonly maxSize: number + + constructor(configuration: WhiskerSearchConfiguration, minSize: number, maxSize: number) { + super(configuration); + this.minSize = minSize; + this.maxSize = maxSize; + } + + /** + * Generate tests by randomly sending events to the Scratch-VM. + * After each Iteration the archive is updated with the trace of executed events. + */ + async generateTests(): Promise { + this._iterations = 0; + this._startTime = Date.now(); + StatisticsCollector.getInstance().iterationCount = 0; + StatisticsCollector.getInstance().coveredFitnessFunctionsCount = 0; + StatisticsCollector.getInstance().startTime = Date.now(); this._fitnessFunctions = this.extractCoverageGoals(); StatisticsCollector.getInstance().fitnessFunctionCount = this._fitnessFunctions.size; this._startTime = Date.now(); - let fullCoverageReached = false; - - const chromosomeGenerator = this._config.getChromosomeGenerator(); const stoppingCondition = this._config.getSearchAlgorithmProperties().getStoppingCondition(); + const eventExtractor = this._config.getEventExtractor(); + const randomTestExecutor = new TestExecutor(Container.vmWrapper, eventExtractor, null); + while (!(stoppingCondition.isFinished(this))) { console.log(`Iteration ${this._iterations}, covered goals: ${this._archive.size}/${this._fitnessFunctions.size}`); + const numberOfEvents = Randomness.getInstance().nextInt(this.minSize, this.maxSize + 1); + const randomEventChromosome = new TestChromosome(new List(), undefined, undefined) + await randomTestExecutor.executeRandomEvents(randomEventChromosome, numberOfEvents); + this.updateArchive(randomEventChromosome); this._iterations++; - StatisticsCollector.getInstance().incrementIterationCount(); - const testChromosome = chromosomeGenerator.get(); - await testChromosome.evaluate(); - this.updateArchive(testChromosome); - if (this._archive.size == this._fitnessFunctions.size && !fullCoverageReached) { - fullCoverageReached = true; - StatisticsCollector.getInstance().createdTestsToReachFullCoverage = this._iterations; - StatisticsCollector.getInstance().timeToReachFullCoverage = Date.now() - this._startTime; - } + this.updateStatistics() } const testSuite = await this.getTestSuite(this._tests); this.collectStatistics(testSuite); @@ -106,6 +130,23 @@ export class RandomTestGenerator extends TestGenerator implements SearchAlgorith } } + /** + * Updates the StatisticsCollector on the following points: + * - bestTestSuiteSize + * - iterationCount + * - createdTestsToReachFullCoverage + * - timeToReachFullCoverage + */ + protected updateStatistics(): void { + StatisticsCollector.getInstance().bestTestSuiteSize = this._tests.size(); + StatisticsCollector.getInstance().incrementIterationCount(); + if (this._archive.size == this._fitnessFunctions.size && !this._fullCoverageReached) { + this._fullCoverageReached = true; + StatisticsCollector.getInstance().createdTestsToReachFullCoverage = this._iterations; + StatisticsCollector.getInstance().timeToReachFullCoverage = Date.now() - this._startTime; + } + } + getCurrentSolution(): List { return this._tests; } diff --git a/whisker-main/src/whisker/utils/WhiskerSearchConfiguration.ts b/whisker-main/src/whisker/utils/WhiskerSearchConfiguration.ts index 54a399d7..3123e48f 100644 --- a/whisker-main/src/whisker/utils/WhiskerSearchConfiguration.ts +++ b/whisker-main/src/whisker/utils/WhiskerSearchConfiguration.ts @@ -78,6 +78,7 @@ export class WhiskerSearchConfiguration { this.dict = Preconditions.checkNotUndefined(dict) } + // TODO: Need variation here; we do not always need all properties, e.g MIO has no crossover public getSearchAlgorithmProperties(): SearchAlgorithmProperties { const populationSize = this.dict['population-size'] as number; const chromosomeLength = this.dict['chromosome-length'] as number; @@ -417,7 +418,7 @@ export class WhiskerSearchConfiguration { public getTestGenerator(): TestGenerator { if (this.dict["test-generator"] == "random") { - return new RandomTestGenerator(this); + return new RandomTestGenerator(this, this.dict['minEventSize'], this.dict['maxEventSize']); } else if (this.dict['test-generator'] == 'iterative') { return new IterativeSearchBasedTestGenerator(this); } else if (this.dict['test-generator'] == 'many-objective') { From de0427934cd27e04837707f806763163a1755c67 Mon Sep 17 00:00:00 2001 From: Patric Feldmeier Date: Thu, 19 Aug 2021 11:45:11 +0200 Subject: [PATCH 6/8] Clarify RandomParameter-Selection in some Events --- config/random.json | 2 +- .../src/whisker/testcase/events/DragSpriteEvent.ts | 2 ++ .../src/whisker/testcase/events/KeyPressEvent.ts | 2 +- .../src/whisker/testcase/events/MouseMoveEvent.ts | 10 +++++----- whisker-main/src/whisker/testcase/events/WaitEvent.ts | 6 +++--- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/config/random.json b/config/random.json index 8b41b1a7..63221d69 100644 --- a/config/random.json +++ b/config/random.json @@ -3,7 +3,7 @@ "algorithm": "random", "minEventSize": 2, "maxEventSize": 30, - "extractor": "static", + "extractor": "naive", "fitness-function": { "type": "statement", "targets": [] diff --git a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts index 1e6e9a8e..302f8ed8 100644 --- a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts +++ b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts @@ -60,6 +60,8 @@ export class DragSpriteEvent extends ScratchEvent { setParameter(args: number[], argType: ParameterType): void { switch (argType) { case ParameterType.RANDOM: + // Arbitrary upper bound aligned to most used codon max value bound. + // 360 is not enough since values above 360 indicate not adding any disturbance at all. this.angle = Randomness.getInstance().nextInt(0, 421); break; case ParameterType.CODON: diff --git a/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts b/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts index 1ab8d387..11c438c1 100644 --- a/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts +++ b/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts @@ -77,7 +77,7 @@ new WaitEvent(this._steps).toJavaScript() setParameter(args:number[], testExecutor:ParameterType): void { switch (testExecutor){ case ParameterType.RANDOM: - this._steps = Randomness.getInstance().nextInt(1, 421); + this._steps = Randomness.getInstance().nextInt(1, Container.config.getPressDurationUpperBound() + 1); break; case ParameterType.CODON: this._steps = args[0]; diff --git a/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts b/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts index 0841c493..45afa2a0 100644 --- a/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts +++ b/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts @@ -70,11 +70,11 @@ export class MouseMoveEvent extends ScratchEvent { switch (argType) { case ParameterType.RANDOM: { const random = Randomness.getInstance(); - const randomX = random.nextInt(0, 421); - const randomY = random.nextInt(0, 361); - const fittedCoordinates = this.fitCoordinates(randomX, randomY); - this._x = fittedCoordinates.x; - this._y = fittedCoordinates.y; + const stageBounds = Container.vmWrapper.getStageSize(); + const signedWidth = stageBounds.width / 2; + const signedHeight = stageBounds.height / 2; + this._x = random.nextInt(-signedWidth, signedWidth + 1); + this._y = random.nextInt(-signedHeight, signedHeight + 1); break; } case ParameterType.CODON: { diff --git a/whisker-main/src/whisker/testcase/events/WaitEvent.ts b/whisker-main/src/whisker/testcase/events/WaitEvent.ts index 0f05b3be..3efec822 100644 --- a/whisker-main/src/whisker/testcase/events/WaitEvent.ts +++ b/whisker-main/src/whisker/testcase/events/WaitEvent.ts @@ -57,10 +57,10 @@ export class WaitEvent extends ScratchEvent { return ["Duration"]; } - setParameter(args:number[], testExecutor:ParameterType): void { - switch (testExecutor){ + setParameter(args: number[], testExecutor: ParameterType): void { + switch (testExecutor) { case ParameterType.RANDOM: - this.steps = Randomness.getInstance().nextInt(0,421); + this.steps = Randomness.getInstance().nextInt(0, Container.config.getWaitStepUpperBound() + 1); break; case ParameterType.CODON: this.steps = args[0]; From 46dd3937cb14e90927547d9bfe7025574eaa1104 Mon Sep 17 00:00:00 2001 From: Patric Feldmeier Date: Thu, 19 Aug 2021 14:37:53 +0200 Subject: [PATCH 7/8] Fix typos and change random assignment of targeted x and y position in DragSpriteEvent --- .../testcase/NaiveScratchEventExtractor.ts | 4 +- .../testcase/StaticScratchEventExtractor.ts | 8 ++- .../src/whisker/testcase/TestExecutor.ts | 2 +- .../testcase/events/DragSpriteEvent.ts | 49 ++++++++++--------- whisker-main/src/whisker/utils/Randomness.ts | 4 +- 5 files changed, 33 insertions(+), 34 deletions(-) diff --git a/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts index 21f121e5..c82466d1 100644 --- a/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts @@ -71,9 +71,7 @@ export class NaiveScratchEventExtractor extends ScratchEventExtractor { // Add events requiring a targets as parameters. for (const target of vm.runtime.targets) { if(!target.isStage) { - const x = this._random.nextInt(-240, 241); - const y = this._random.nextInt(-180, 181); - eventList.add(new DragSpriteEvent(target, x, y)); + eventList.add(new DragSpriteEvent(target)); eventList.add(new ClickSpriteEvent(target)); } } diff --git a/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts index 693f248e..ecf38be8 100644 --- a/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts @@ -37,7 +37,7 @@ import {TypeTextEvent} from "./events/TypeTextEvent"; export class StaticScratchEventExtractor extends ScratchEventExtractor { // TODO: Additional keys? - private readonly KEYS = ['space', 'left arrow', 'up arrow', 'right arrow', 'down arrow', 'enter']; + private readonly _keys = ['space', 'left arrow', 'up arrow', 'right arrow', 'down arrow', 'enter']; private readonly _random: Randomness; @@ -88,7 +88,7 @@ export class StaticScratchEventExtractor extends ScratchEventExtractor { case 'sensing_keypressed': { // Only add if we have not yet found any keyPress-Events. No need to add all keys several times if (!eventList.getElements().some(event => event instanceof KeyPressEvent)) { - for (const key of this.KEYS) { + for (const key of this._keys) { eventList.add(new KeyPressEvent(key)); } } @@ -103,9 +103,7 @@ export class StaticScratchEventExtractor extends ScratchEventExtractor { } case 'sensing_touchingobject': case 'sensing_touchingcolor' : { - const x = this._random.nextInt(-240, 241); - const y = this._random.nextInt(-180, 181); - eventList.add(new DragSpriteEvent(target, x, y)); + eventList.add(new DragSpriteEvent(target)); break; } case 'sensing_distanceto': { diff --git a/whisker-main/src/whisker/testcase/TestExecutor.ts b/whisker-main/src/whisker/testcase/TestExecutor.ts index 5df9b41b..f488d86c 100644 --- a/whisker-main/src/whisker/testcase/TestExecutor.ts +++ b/whisker-main/src/whisker/testcase/TestExecutor.ts @@ -163,7 +163,7 @@ export class TestExecutor { const event = availableEvents.get(eventIndex); eventCount++; - // If the selected required additional parameters; select them randomly as well. + // If the selected event requires additional parameters; select them randomly as well. if (event.numSearchParameter() > 0) { // args are set in the event itself since the event knows which range of random values makes sense. event.setParameter(null, ParameterType.RANDOM); diff --git a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts index 302f8ed8..f9d843f8 100644 --- a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts +++ b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts @@ -59,34 +59,37 @@ export class DragSpriteEvent extends ScratchEvent { setParameter(args: number[], argType: ParameterType): void { switch (argType) { - case ParameterType.RANDOM: - // Arbitrary upper bound aligned to most used codon max value bound. - // 360 is not enough since values above 360 indicate not adding any disturbance at all. - this.angle = Randomness.getInstance().nextInt(0, 421); + case ParameterType.RANDOM: { + const random = Randomness.getInstance(); + const stageBounds = Container.vmWrapper.getStageSize(); + const signedWidth = stageBounds.width / 2; + const signedHeight = stageBounds.height / 2; + this._x = random.nextInt(-signedWidth, signedWidth + 1); + this._y = random.nextInt(-signedHeight, signedHeight + 1); break; + } case ParameterType.CODON: this.angle = args[0]; + // We only disturb the target point if we have an angle smaller than 360 degrees. + if (this.angle < 360) { + // Convert to Radians and fetch the sprite's horizontal and vertical size. + const radians = this.angle / 180 * Math.PI; + const bounds = this._target.getBounds(); + const horizontalSize = Math.abs(bounds.right - bounds.left); + const verticalSize = Math.abs(bounds.top - bounds.bottom); + + // Calculate the distorted position. + const stageWidth = Container.vmWrapper.getStageSize().width / 2; + const stageHeight = Container.vmWrapper.getStageSize().height / 2; + this._x += horizontalSize * Math.cos(radians); + this._y += verticalSize * Math.sin(radians); + + // Clamp the new position within the stage size + this._x = Math.max(-stageWidth, Math.min(this._x, stageWidth)); + this._y = Math.max(-stageHeight, Math.min(this._y, stageHeight)); + } break; } - - // We only disturb the target point if we have an angle smaller than 360 degrees. - if (this.angle < 360) { - // Convert to Radians and fetch the sprite's horizontal and vertical size. - const radians = this.angle / 180 * Math.PI; - const bounds = this._target.getBounds(); - const horizontalSize = Math.abs(bounds.right - bounds.left); - const verticalSize = Math.abs(bounds.top - bounds.bottom); - - // Calculate the distorted position. - const stageWidth = Container.vmWrapper.getStageSize().width / 2; - const stageHeight = Container.vmWrapper.getStageSize().height / 2; - this._x += horizontalSize * Math.cos(radians); - this._y += verticalSize * Math.sin(radians); - - // Clamp the new position within the stage size - this._x = Math.max(-stageWidth, Math.min(this._x, stageWidth)); - this._y = Math.max(-stageHeight, Math.min(this._y, stageHeight)); - } } numSearchParameter(): number { diff --git a/whisker-main/src/whisker/utils/Randomness.ts b/whisker-main/src/whisker/utils/Randomness.ts index bf2f9a53..196af558 100644 --- a/whisker-main/src/whisker/utils/Randomness.ts +++ b/whisker-main/src/whisker/utils/Randomness.ts @@ -84,8 +84,8 @@ export class Randomness { /** * Pick a random integer from a range - * @param min Lower bound of range, included - * @param max Upper bound of range, excluded + * @param min Lower bound of range inclusive + * @param max Upper bound of range exclusive */ public nextInt(min: number, max: number): number { return Math.floor(this.next(min, max)); From d006c9ae6427c05c797204add749f0535bfabe86 Mon Sep 17 00:00:00 2001 From: Patric Feldmeier Date: Thu, 19 Aug 2021 18:19:55 +0200 Subject: [PATCH 8/8] Clarify DragSpriteEvents --- whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts index f9d843f8..4882d6e5 100644 --- a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts +++ b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts @@ -68,6 +68,10 @@ export class DragSpriteEvent extends ScratchEvent { this._y = random.nextInt(-signedHeight, signedHeight + 1); break; } + // When using codons, we sometimes want to slightly disturb the position wo which the target should + // be dragged to since the position could have some with unintended side effects. Hence, we disturb the + // given position with a power equal to the target's size. The direction of the disturbance is determined + // through a codon. If we have a codon value above 360, the position is not disturbed at all. case ParameterType.CODON: this.angle = args[0]; // We only disturb the target point if we have an angle smaller than 360 degrees.