diff --git a/config/random.json b/config/random.json new file mode 100644 index 00000000..63221d69 --- /dev/null +++ b/config/random.json @@ -0,0 +1,42 @@ +{ + "test-generator": "random", + "algorithm": "random", + "minEventSize": 2, + "maxEventSize": 30, + "extractor": "naive", + "fitness-function": { + "type": "statement", + "targets": [] + }, + "stopping-condition": { + "type": "one-of", + "conditions": [ + { + "type": "fixed-time", + "duration": 6000000 + }, + { + "type": "optimal" + } + ] + }, + "crossover": { + "operator": "singlepointrelative", + "probability": 1 + }, + "mutation": { + "operator": "biasedVariablelengthConstrained", + "probability": 1, + "gaussianMutationPower": 5, + "maxMutationCountStart": 0, + "maxMutationCountFocusedPhase": 10 + }, + "selection": { + "operator": "rank", + "randomSelectionProbabilityStart": 0.5, + "randomSelectionProbabilityFocusedPhase": 0 + }, + "waitStepUpperBound": 200, + "click-duration": 3, + "press-duration": 50 +} diff --git a/whisker-main/src/whisker/search/algorithms/SimpleGA.ts b/whisker-main/src/whisker/search/algorithms/SimpleGA.ts index d6c12aa4..03daa0ba 100644 --- a/whisker-main/src/whisker/search/algorithms/SimpleGA.ts +++ b/whisker-main/src/whisker/search/algorithms/SimpleGA.ts @@ -97,8 +97,10 @@ export class SimpleGA extends SearchAlgorithmDefault { let population = this.generateInitialPopulation(); await this.evaluatePopulation(population); - // Evaluate population - this.evaluateAndSortPopulation(population); + // Evaluate population, but before check if we have already reached our stopping condition + if(!(this._stoppingCondition.isFinished(this))) { + this.evaluateAndSortPopulation(population); + } while (!(this._stoppingCondition.isFinished(this))) { console.log(`Iteration ${this._iterations}, best fitness: ${this._bestFitness}`); @@ -106,7 +108,7 @@ export class SimpleGA extends SearchAlgorithmDefault { const nextGeneration = this.generateOffspringPopulation(population); await this.evaluatePopulation(nextGeneration); if(!(this._stoppingCondition.isFinished(this))) { - this.evaluateAndSortPopulation(nextGeneration) + this.evaluateAndSortPopulation(nextGeneration); } population = nextGeneration; this._iterations++; diff --git a/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts index 3d5f636a..c82466d1 100644 --- a/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/NaiveScratchEventExtractor.ts @@ -27,37 +27,53 @@ import {ScratchEventExtractor} from "./ScratchEventExtractor"; import {MouseDownEvent} from "./events/MouseDownEvent"; import {MouseMoveEvent} from "./events/MouseMoveEvent"; import {KeyPressEvent} from "./events/KeyPressEvent"; -import {SoundEvent} from "./events/SoundEvent"; import {TypeTextEvent} from "./events/TypeTextEvent"; import {DragSpriteEvent} from "./events/DragSpriteEvent"; +import {ClickSpriteEvent} from "./events/ClickSpriteEvent"; +import {ClickStageEvent} from "./events/ClickStageEvent"; +import {Randomness} from "../utils/Randomness"; +import {SoundEvent} from "./events/SoundEvent"; export class NaiveScratchEventExtractor extends ScratchEventExtractor { + // TODO: Additional keys? private readonly KEYS = ['space', 'left arrow', 'up arrow', 'right arrow', 'down arrow', 'enter']; - private readonly _text; + private readonly _random: Randomness; + /** + * NaiveScratchEventExtractor adds every type of supported Whisker-Event to the set of avilalbe events. + * Whenever a parameter is required, it is randomly selected. + * @param vm the Scratch-VM + */ constructor(vm: VirtualMachine) { super(vm); - this._text = this._randomText(3); + this._random = Randomness.getInstance(); } public extractEvents(vm: VirtualMachine): List { const eventList = new List(); + eventList.add(new ClickStageEvent()); eventList.add(new WaitEvent()); - eventList.add(new TypeTextEvent(this._text)); - eventList.addList(this._getTypeTextEvents()); // Just one random string + eventList.add(new TypeTextEvent(this._randomText(3))); eventList.add(new MouseDownEvent(true)); eventList.add(new MouseDownEvent(false)); eventList.add(new MouseMoveEvent()); - // eventList.add(new SoundEvent()); // Not implemented yet + // eventList.add(new SoundEvent()) not implemented yet + + // Add specified keys. for (const key of this.KEYS) { eventList.add(new KeyPressEvent(key)); } + + // Add events requiring a targets as parameters. for (const target of vm.runtime.targets) { - eventList.add(new DragSpriteEvent(target)); + if(!target.isStage) { + eventList.add(new DragSpriteEvent(target)); + eventList.add(new ClickSpriteEvent(target)); + } } return eventList.distinctObjects(); } diff --git a/whisker-main/src/whisker/testcase/ScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/ScratchEventExtractor.ts index e6d0c003..a806ecee 100644 --- a/whisker-main/src/whisker/testcase/ScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/ScratchEventExtractor.ts @@ -101,8 +101,6 @@ export abstract class ScratchEventExtractor { } } - - // TODO: How to handle event parameters? protected _extractEventsFromBlock(target, block): List { const eventList = new List(); if (typeof block.opcode === 'undefined') { @@ -123,7 +121,8 @@ export abstract class ScratchEventExtractor { break; } case 'sensing_mousex': - case 'sensing_mousey': { + case 'sensing_mousey': + case 'pen_penDown': { // Mouse move eventList.add(new MouseMoveEvent()); break; @@ -190,7 +189,6 @@ export abstract class ScratchEventExtractor { const field = target.blocks.getFields(distanceMenuBlock); const value = field.DISTANCETOMENU.value; if (value == "_mouse_") { - // TODO: Maybe could determine position to move to here? eventList.add(new MouseMoveEvent()); } break; @@ -207,10 +205,6 @@ export abstract class ScratchEventExtractor { eventList.add(new MouseDownEvent(!isMouseDown)); break; } - case 'pen_penDown': { - eventList.add(new MouseMoveEvent()) - break; - } case 'sensing_askandwait': // Type text if (Container.vmWrapper.isQuestionAsked()) { diff --git a/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts b/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts index b6aaad7b..ecf38be8 100644 --- a/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts +++ b/whisker-main/src/whisker/testcase/StaticScratchEventExtractor.ts @@ -24,11 +24,32 @@ import VirtualMachine from 'scratch-vm/src/virtual-machine.js'; import {ScratchEvent} from "./events/ScratchEvent"; import {WaitEvent} from "./events/WaitEvent"; import {ScratchEventExtractor} from "./ScratchEventExtractor"; +import {KeyPressEvent} from "./events/KeyPressEvent"; +import {MouseMoveEvent} from "./events/MouseMoveEvent"; +import {Randomness} from "../utils/Randomness"; +import {DragSpriteEvent} from "./events/DragSpriteEvent"; +import {MouseDownEvent} from "./events/MouseDownEvent"; +import {ClickSpriteEvent} from "./events/ClickSpriteEvent"; +import {ClickStageEvent} from "./events/ClickStageEvent"; +import {SoundEvent} from "./events/SoundEvent"; +import {TypeTextEvent} from "./events/TypeTextEvent"; export class StaticScratchEventExtractor extends ScratchEventExtractor { - constructor (vm: VirtualMachine) { + // TODO: Additional keys? + private readonly _keys = ['space', 'left arrow', 'up arrow', 'right arrow', 'down arrow', 'enter']; + + private readonly _random: Randomness; + + /** + * StaticScratchEventExtractor only adds event for which corresponding event handler exist. However, as opposed + * to the DynamicScratchEventExtractor, the parameters are chosen randomly and not inferred from the VM whenever + * possible. + * @param vm the Scratch-VM + */ + constructor(vm: VirtualMachine) { super(vm); + this._random = Randomness.getInstance(); } public extractEvents(vm: VirtualMachine): List { @@ -48,4 +69,81 @@ export class StaticScratchEventExtractor extends ScratchEventExtractor { return eventList.distinctObjects(); } + + /** + * Extract events for which corresponding event handler exist. The parameter of the selected events are chosen + * randomly. + * @param target the sprite wrapped in a RenderedTarget + * @param block the given block from which events might get extracted. + * @returns List containing all extracted events. + */ + protected _extractEventsFromBlock(target, block): List { + const eventList = new List(); + if (typeof block.opcode === 'undefined') { + return eventList; + } + + switch (target.blocks.getOpcode(block)) { + case 'event_whenkeypressed': + case 'sensing_keypressed': { + // Only add if we have not yet found any keyPress-Events. No need to add all keys several times + if (!eventList.getElements().some(event => event instanceof KeyPressEvent)) { + for (const key of this._keys) { + eventList.add(new KeyPressEvent(key)); + } + } + break; + } + case 'sensing_mousex': + case 'sensing_mousey': + case 'pen_penDown': { + // Mouse move + eventList.add(new MouseMoveEvent()); + break; + } + case 'sensing_touchingobject': + case 'sensing_touchingcolor' : { + eventList.add(new DragSpriteEvent(target)); + break; + } + case 'sensing_distanceto': { + const distanceMenuBlock = target.blocks.getBlock(block.inputs.DISTANCETOMENU.block); + const field = target.blocks.getFields(distanceMenuBlock); + const value = field.DISTANCETOMENU.value; + if (value == "_mouse_") { + eventList.add(new MouseMoveEvent()); + } + break; + } + case 'motion_pointtowards': { + const towards = target.blocks.getBlock(block.inputs.TOWARDS.block) + if (towards.fields.TOWARDS.value === '_mouse_') + eventList.add(new MouseMoveEvent()); + break; + } + case 'sensing_mousedown': { + // Mouse down + eventList.add(new MouseDownEvent(true)); + eventList.add(new MouseDownEvent(false)); + break; + } + case 'sensing_askandwait': + // Type text + eventList.add(new TypeTextEvent(this._randomText(3))); + break; + case 'event_whenthisspriteclicked': + // Click sprite + eventList.add(new ClickSpriteEvent(target)); + break; + case 'event_whenstageclicked': + // Click stage + eventList.add(new ClickStageEvent()); + break; + case 'event_whengreaterthan': + // Sound + eventList.add(new SoundEvent()); + break; + } + return eventList; + } } diff --git a/whisker-main/src/whisker/testcase/TestExecutor.ts b/whisker-main/src/whisker/testcase/TestExecutor.ts index e8208c30..f488d86c 100644 --- a/whisker-main/src/whisker/testcase/TestExecutor.ts +++ b/whisker-main/src/whisker/testcase/TestExecutor.ts @@ -29,11 +29,11 @@ import {StatisticsCollector} from "../utils/StatisticsCollector"; import {EventObserver} from "./EventObserver"; import {seedScratch} from "../../util/random"; import {Randomness} from "../utils/Randomness"; -import VMWrapper = require("../../vm/vm-wrapper.js") import {ScratchEventExtractor} from "./ScratchEventExtractor"; import Runtime from "scratch-vm/src/engine/runtime"; import {EventSelector} from "./EventSelector"; -import {ParameterTypes} from "./events/ParameterTypes"; +import {ParameterType} from "./events/ParameterType"; +import VMWrapper = require("../../vm/vm-wrapper.js"); export class TestExecutor { @@ -62,6 +62,10 @@ export class TestExecutor { } } + /** + * Executes a chromosome by selecting events according to the chromosome's defined genes. + * @param testChromosome the testChromosome that should be executed. + */ async execute(testChromosome: TestChromosome): Promise { const events = new List<[ScratchEvent, number[]]>(); @@ -81,7 +85,6 @@ export class TestExecutor { while (numCodon < codons.size() && (this._projectRunning || this.hasActionEvents(availableEvents))) { availableEvents = this._eventExtractor.extractEvents(this._vm); - if (availableEvents.isEmpty()) { console.log("Whisker-Main: No events available for project."); break; @@ -100,11 +103,11 @@ export class TestExecutor { // Check if we came closer to cover a specific block. This is only makes sense when using a SingleObjective // focused Algorithm like MIO. - if(testChromosome.targetFitness){ + if (testChromosome.targetFitness) { testChromosome.trace = new ExecutionTrace(this._vm.runtime.traceInfo.tracer.traces, events.clone()); testChromosome.coverage = currentCoverage; const currentFitness = testChromosome.targetFitness.getFitness(testChromosome); - if(testChromosome.targetFitness.compare(currentFitness, targetFitness) > 0){ + if (testChromosome.targetFitness.compare(currentFitness, targetFitness) > 0) { targetFitness = currentFitness; testChromosome.lastImprovedFitnessCodon = numCodon; } @@ -131,6 +134,59 @@ export class TestExecutor { return testChromosome.trace; } + /** + * Randomly executes events selected from the available event Set. + * @param randomEventChromosome the chromosome in which the executed trace will be saved in. + * @param numberOfEvents the number of events that should be executed. + */ + async executeRandomEvents(randomEventChromosome: TestChromosome, numberOfEvents: number): Promise { + seedScratch(String(Randomness.getInitialSeed())); + const _onRunStop = this.projectStopped.bind(this); + this._vm.on(Runtime.PROJECT_RUN_STOP, _onRunStop); + this._projectRunning = true; + this._vmWrapper.start(); + let availableEvents = this._eventExtractor.extractEvents(this._vm); + let eventCount = 0; + const random = Randomness.getInstance(); + const events = new List<[ScratchEvent, number[]]>(); + + while (eventCount < numberOfEvents && (this._projectRunning || this.hasActionEvents(availableEvents))) { + availableEvents = this._eventExtractor.extractEvents(this._vm); + if (availableEvents.isEmpty()) { + console.log("Whisker-Main: No events available for project."); + break; + } + + // Randomly select an event and increase the event count. + const eventIndex = random.nextInt(0, availableEvents.size()); + randomEventChromosome.getGenes().add(eventIndex); + const event = availableEvents.get(eventIndex); + eventCount++; + + // If the selected event requires additional parameters; select them randomly as well. + if (event.numSearchParameter() > 0) { + // args are set in the event itself since the event knows which range of random values makes sense. + event.setParameter(null, ParameterType.RANDOM); + } + events.add([event, event.getParameter()]); + await event.apply(); + StatisticsCollector.getInstance().incrementEventsCount() + + // Send a WaitEvent to the VM + const waitEvent = new WaitEvent(1); + events.add([waitEvent, []]); + await waitEvent.apply(); + + } + const trace = new ExecutionTrace(this._vm.runtime.traceInfo.tracer.traces, events); + randomEventChromosome.coverage = this._vm.runtime.traceInfo.tracer.coverage as Set; + randomEventChromosome.trace = trace; + this._vmWrapper.end(); + this.resetState(); + StatisticsCollector.getInstance().numberFitnessEvaluations++; + return trace; + } + /** * Selects and sends the next Event to the VM * @param codons the list of codons deciding which event and parameters to take @@ -145,9 +201,9 @@ export class TestExecutor { const nextEvent: ScratchEvent = this._eventSelector.selectEvent(codons, numCodon, availableEvents); numCodon++; const args = TestExecutor.getArgs(nextEvent, codons, numCodon); - nextEvent.setParameter(args, ParameterTypes.CODON); + nextEvent.setParameter(args, ParameterType.CODON); events.add([nextEvent, args]); - numCodon += nextEvent.getNumVariableParameters(); + numCodon += nextEvent.numSearchParameter(); this.notify(nextEvent, args); // Send the chosen Event including its parameters to the VM await nextEvent.apply(); @@ -168,7 +224,7 @@ export class TestExecutor { */ private static getArgs(event: ScratchEvent, codons: List, codonPosition: number): number[] { const args = []; - for (let i = 0; i < event.getNumVariableParameters(); i++) { + for (let i = 0; i < event.numSearchParameter(); i++) { args.push(codons.get(codonPosition++ % codons.size())); } return args; diff --git a/whisker-main/src/whisker/testcase/events/ClickSpriteEvent.ts b/whisker-main/src/whisker/testcase/events/ClickSpriteEvent.ts index e3136f76..d8744727 100644 --- a/whisker-main/src/whisker/testcase/events/ClickSpriteEvent.ts +++ b/whisker-main/src/whisker/testcase/events/ClickSpriteEvent.ts @@ -81,7 +81,7 @@ export class ClickSpriteEvent extends ScratchEvent { } } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } @@ -93,7 +93,7 @@ export class ClickSpriteEvent extends ScratchEvent { return [this._target, this._steps]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/ClickStageEvent.ts b/whisker-main/src/whisker/testcase/events/ClickStageEvent.ts index 76cd0a91..cf9a7f53 100644 --- a/whisker-main/src/whisker/testcase/events/ClickStageEvent.ts +++ b/whisker-main/src/whisker/testcase/events/ClickStageEvent.ts @@ -37,7 +37,7 @@ export class ClickStageEvent extends ScratchEvent { return "ClickStage" } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } @@ -49,7 +49,7 @@ export class ClickStageEvent extends ScratchEvent { return []; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts index 12d215b6..4882d6e5 100644 --- a/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts +++ b/whisker-main/src/whisker/testcase/events/DragSpriteEvent.ts @@ -21,6 +21,8 @@ import {ScratchEvent} from "./ScratchEvent"; import {RenderedTarget} from 'scratch-vm/src/sprites/rendered-target'; import {Container} from "../../utils/Container"; +import {ParameterType} from "./ParameterType"; +import {Randomness} from "../../utils/Randomness"; export class DragSpriteEvent extends ScratchEvent { @@ -55,34 +57,50 @@ export class DragSpriteEvent extends ScratchEvent { return [this._x, this._y, this.angle, this._target.sprite.name]; } - setParameter(args: number[]): void { - this.angle = args[0]; - - // We only disturb the target point if we have an angle smaller than 360 degrees. - if (this.angle < 360) { - // Convert to Radians and fetch the sprite's horizontal and vertical size. - const radians = this.angle / 180 * Math.PI; - const bounds = this._target.getBounds(); - const horizontalSize = Math.abs(bounds.right - bounds.left); - const verticalSize = Math.abs(bounds.top - bounds.bottom); - - // Calculate the distorted position. - const stageWidth = Container.vmWrapper.getStageSize().width / 2; - const stageHeight = Container.vmWrapper.getStageSize().height / 2; - this._x += horizontalSize * Math.cos(radians); - this._y += verticalSize * Math.sin(radians); - - // Clamp the new position within the stage size - this._x = Math.max(-stageWidth, Math.min(this._x, stageWidth)); - this._y = Math.max(-stageHeight, Math.min(this._y, stageHeight)); + setParameter(args: number[], argType: ParameterType): void { + switch (argType) { + case ParameterType.RANDOM: { + const random = Randomness.getInstance(); + const stageBounds = Container.vmWrapper.getStageSize(); + const signedWidth = stageBounds.width / 2; + const signedHeight = stageBounds.height / 2; + this._x = random.nextInt(-signedWidth, signedWidth + 1); + this._y = random.nextInt(-signedHeight, signedHeight + 1); + break; + } + // When using codons, we sometimes want to slightly disturb the position wo which the target should + // be dragged to since the position could have some with unintended side effects. Hence, we disturb the + // given position with a power equal to the target's size. The direction of the disturbance is determined + // through a codon. If we have a codon value above 360, the position is not disturbed at all. + case ParameterType.CODON: + this.angle = args[0]; + // We only disturb the target point if we have an angle smaller than 360 degrees. + if (this.angle < 360) { + // Convert to Radians and fetch the sprite's horizontal and vertical size. + const radians = this.angle / 180 * Math.PI; + const bounds = this._target.getBounds(); + const horizontalSize = Math.abs(bounds.right - bounds.left); + const verticalSize = Math.abs(bounds.top - bounds.bottom); + + // Calculate the distorted position. + const stageWidth = Container.vmWrapper.getStageSize().width / 2; + const stageHeight = Container.vmWrapper.getStageSize().height / 2; + this._x += horizontalSize * Math.cos(radians); + this._y += verticalSize * Math.sin(radians); + + // Clamp the new position within the stage size + this._x = Math.max(-stageWidth, Math.min(this._x, stageWidth)); + this._y = Math.max(-stageHeight, Math.min(this._y, stageHeight)); + } + break; } } - getNumVariableParameters(): number { + numSearchParameter(): number { return 1; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts b/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts index 3790f024..11c438c1 100644 --- a/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts +++ b/whisker-main/src/whisker/testcase/events/KeyPressEvent.ts @@ -21,8 +21,9 @@ import {ScratchEvent} from "./ScratchEvent"; import {Container} from "../../utils/Container"; import {WaitEvent} from "./WaitEvent"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; import {NeuroevolutionUtil} from "../../whiskerNet/NeuroevolutionUtil"; +import {Randomness} from "../../utils/Randomness"; export class KeyPressEvent extends ScratchEvent { @@ -61,7 +62,7 @@ new WaitEvent(this._steps).toJavaScript() return "KeyPress " + this._keyOption + ": " + this._steps; } - getNumVariableParameters(): number { + numSearchParameter(): number { return 1; } @@ -69,16 +70,19 @@ new WaitEvent(this._steps).toJavaScript() return [this._keyOption, this._steps]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return ["Steps"]; } - setParameter(args:number[], testExecutor:ParameterTypes): void { + setParameter(args:number[], testExecutor:ParameterType): void { switch (testExecutor){ - case ParameterTypes.CODON: + case ParameterType.RANDOM: + this._steps = Randomness.getInstance().nextInt(1, Container.config.getPressDurationUpperBound() + 1); + break; + case ParameterType.CODON: this._steps = args[0]; break; - case ParameterTypes.REGRESSION: + case ParameterType.REGRESSION: this._steps = Math.round(NeuroevolutionUtil.relu(args[0])); break; } diff --git a/whisker-main/src/whisker/testcase/events/MouseDownEvent.ts b/whisker-main/src/whisker/testcase/events/MouseDownEvent.ts index d8bc7c2b..660a870c 100644 --- a/whisker-main/src/whisker/testcase/events/MouseDownEvent.ts +++ b/whisker-main/src/whisker/testcase/events/MouseDownEvent.ts @@ -48,7 +48,7 @@ export class MouseDownEvent extends ScratchEvent { return "MouseDown " + this._value; } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } @@ -57,7 +57,7 @@ export class MouseDownEvent extends ScratchEvent { return [this._value ? 1 : 0]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts b/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts index 973951f1..45afa2a0 100644 --- a/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts +++ b/whisker-main/src/whisker/testcase/events/MouseMoveEvent.ts @@ -20,7 +20,8 @@ import {ScratchEvent} from "./ScratchEvent"; import {Container} from "../../utils/Container"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; +import {Randomness} from "../../utils/Randomness"; export class MouseMoveEvent extends ScratchEvent { @@ -53,7 +54,7 @@ export class MouseMoveEvent extends ScratchEvent { return "MouseMove " + Math.trunc(this._x) + "/" + Math.trunc(this._y); } - getNumVariableParameters(): number { + numSearchParameter(): number { return 2; // x and y? } @@ -61,19 +62,28 @@ export class MouseMoveEvent extends ScratchEvent { return [this._x, this._y]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return ["X", "Y"] } - setParameter(args: number[], argType: ParameterTypes): void { + setParameter(args: number[], argType: ParameterType): void { switch (argType) { - case ParameterTypes.CODON: { + case ParameterType.RANDOM: { + const random = Randomness.getInstance(); + const stageBounds = Container.vmWrapper.getStageSize(); + const signedWidth = stageBounds.width / 2; + const signedHeight = stageBounds.height / 2; + this._x = random.nextInt(-signedWidth, signedWidth + 1); + this._y = random.nextInt(-signedHeight, signedHeight + 1); + break; + } + case ParameterType.CODON: { const fittedCoordinates = this.fitCoordinates(args[0], args[1]) this._x = fittedCoordinates.x; this._y = fittedCoordinates.y; break; } - case ParameterTypes.REGRESSION: { + case ParameterType.REGRESSION: { this._x = Math.tanh(args[0]) * 240; this._y = Math.tanh(args[1]) * 180; break; diff --git a/whisker-main/src/whisker/testcase/events/MouseMoveToEvent.ts b/whisker-main/src/whisker/testcase/events/MouseMoveToEvent.ts index 17770c29..ff87c308 100644 --- a/whisker-main/src/whisker/testcase/events/MouseMoveToEvent.ts +++ b/whisker-main/src/whisker/testcase/events/MouseMoveToEvent.ts @@ -53,7 +53,7 @@ export class MouseMoveToEvent extends ScratchEvent { return "MouseMove " + Math.trunc(this.x) + "/" + Math.trunc(this.y); } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } @@ -61,7 +61,7 @@ export class MouseMoveToEvent extends ScratchEvent { return [this.x, this.y]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/ParameterType.ts b/whisker-main/src/whisker/testcase/events/ParameterType.ts new file mode 100644 index 00000000..7b9aa75e --- /dev/null +++ b/whisker-main/src/whisker/testcase/events/ParameterType.ts @@ -0,0 +1,7 @@ +export enum ParameterType { + "RANDOM", + + "CODON", + + "REGRESSION" +} diff --git a/whisker-main/src/whisker/testcase/events/ParameterTypes.ts b/whisker-main/src/whisker/testcase/events/ParameterTypes.ts deleted file mode 100644 index e021111f..00000000 --- a/whisker-main/src/whisker/testcase/events/ParameterTypes.ts +++ /dev/null @@ -1,5 +0,0 @@ -export enum ParameterTypes { - "CODON", - - "REGRESSION" -} diff --git a/whisker-main/src/whisker/testcase/events/ScratchEvent.ts b/whisker-main/src/whisker/testcase/events/ScratchEvent.ts index 3e1ac8da..05c3cc64 100644 --- a/whisker-main/src/whisker/testcase/events/ScratchEvent.ts +++ b/whisker-main/src/whisker/testcase/events/ScratchEvent.ts @@ -20,7 +20,7 @@ import {RenderedTarget} from 'scratch-vm/src/sprites/rendered-target'; import {Container} from "../../utils/Container"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; export abstract class ScratchEvent { @@ -31,26 +31,26 @@ export abstract class ScratchEvent { abstract apply(): Promise; /** - * Returns the number of variable parameters required by this event. + * Returns the number of parameters that will be defined during search. */ - abstract getNumVariableParameters(): number; + abstract numSearchParameter(): number; /** - * Sets the parameter(s) of this event using the given arguments. - * @param args the values to which the parameters of this event should be set to - * @param argType the type of the given arguments decides how they should be interpreted as parameters. + * Returns the name(s) of parameter(s) defined during search. */ - abstract setParameter(args: number[], argType: ParameterTypes): void; + abstract getSearchParameterNames(): string[]; /** - * Returns the parameter(s) of this event. + * Sets the parameter(s) of this event using the given arguments. + * @param args the values to which the parameters of this event should be set to + * @param argType the type of the given arguments decide how they should be interpreted as parameters. */ - abstract getParameter(): (number | string | RenderedTarget) []; + abstract setParameter(args: number[], argType: ParameterType): void; /** - * Returns the name(s) of variable parameter(s). + * Returns all parameter(s) of this event. */ - abstract getVariableParameterNames(): string[]; + abstract getParameter(): (number | string | RenderedTarget) []; /** * Transforms the event into an executable Whisker-Test statement. @@ -63,9 +63,9 @@ export abstract class ScratchEvent { abstract toString(): string; /** - * Returns an identifier as string. Events containing variable parameters obtain the same identifier and - * Events whose parameters are determined by the ScratchEventExtractor get different identifiers. - * The id is used to query the right RegressionNode if variable parameters for a specific Event are needed. + * Returns an identifier as string. Events containing parameters defined during search obtain the same + * identifier and Events whose parameters are determined by the ScratchEventExtractor get different identifiers. + * The id is used to query the right RegressionNode if search defined parameters for a specific Event are needed. */ abstract stringIdentifier():string; diff --git a/whisker-main/src/whisker/testcase/events/SoundEvent.ts b/whisker-main/src/whisker/testcase/events/SoundEvent.ts index 2c728b2f..f7397338 100644 --- a/whisker-main/src/whisker/testcase/events/SoundEvent.ts +++ b/whisker-main/src/whisker/testcase/events/SoundEvent.ts @@ -37,7 +37,7 @@ export class SoundEvent extends ScratchEvent { throw new NotYetImplementedException(); } - getNumVariableParameters(): number { + numSearchParameter(): number { return 1; } @@ -45,7 +45,7 @@ export class SoundEvent extends ScratchEvent { return [this._volume]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return ["Volume"]; } diff --git a/whisker-main/src/whisker/testcase/events/TypeTextEvent.ts b/whisker-main/src/whisker/testcase/events/TypeTextEvent.ts index f6ded097..6feae01d 100644 --- a/whisker-main/src/whisker/testcase/events/TypeTextEvent.ts +++ b/whisker-main/src/whisker/testcase/events/TypeTextEvent.ts @@ -48,7 +48,7 @@ export class TypeTextEvent extends ScratchEvent { return `TypeText '${this._text}'` } - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; // Text } @@ -56,7 +56,7 @@ export class TypeTextEvent extends ScratchEvent { return [this._text]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } diff --git a/whisker-main/src/whisker/testcase/events/WaitEvent.ts b/whisker-main/src/whisker/testcase/events/WaitEvent.ts index 40642bc6..3efec822 100644 --- a/whisker-main/src/whisker/testcase/events/WaitEvent.ts +++ b/whisker-main/src/whisker/testcase/events/WaitEvent.ts @@ -20,8 +20,9 @@ import {ScratchEvent} from "./ScratchEvent"; import {Container} from "../../utils/Container"; -import {ParameterTypes} from "./ParameterTypes"; +import {ParameterType} from "./ParameterType"; import {NeuroevolutionUtil} from "../../whiskerNet/NeuroevolutionUtil"; +import {Randomness} from "../../utils/Randomness"; export class WaitEvent extends ScratchEvent { @@ -44,7 +45,7 @@ export class WaitEvent extends ScratchEvent { return "Wait for " + this.steps + " steps"; } - getNumVariableParameters(): number { + numSearchParameter(): number { return 1; } @@ -52,16 +53,19 @@ export class WaitEvent extends ScratchEvent { return [this.steps]; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return ["Duration"]; } - setParameter(args:number[], testExecutor:ParameterTypes): void { - switch (testExecutor){ - case ParameterTypes.CODON: + setParameter(args: number[], testExecutor: ParameterType): void { + switch (testExecutor) { + case ParameterType.RANDOM: + this.steps = Randomness.getInstance().nextInt(0, Container.config.getWaitStepUpperBound() + 1); + break; + case ParameterType.CODON: this.steps = args[0]; break; - case ParameterTypes.REGRESSION: + case ParameterType.REGRESSION: this.steps = Math.round(NeuroevolutionUtil.relu(args[0])); break; } diff --git a/whisker-main/src/whisker/testgenerator/RandomTestGenerator.ts b/whisker-main/src/whisker/testgenerator/RandomTestGenerator.ts index 9c18ffaf..5220f10b 100644 --- a/whisker-main/src/whisker/testgenerator/RandomTestGenerator.ts +++ b/whisker-main/src/whisker/testgenerator/RandomTestGenerator.ts @@ -19,23 +19,21 @@ */ import {TestGenerator} from './TestGenerator'; -import {ScratchProject} from '../scratch/ScratchProject'; import {List} from '../utils/List'; -import {SearchAlgorithmProperties} from '../search/SearchAlgorithmProperties'; -import {ChromosomeGenerator} from "../search/ChromosomeGenerator"; import {TestChromosome} from "../testcase/TestChromosome"; import {SearchAlgorithm} from "../search/SearchAlgorithm"; -import {Selection} from "../search/Selection"; import {NotSupportedFunctionException} from "../core/exceptions/NotSupportedFunctionException"; import {FitnessFunction} from "../search/FitnessFunction"; import {StatisticsCollector} from "../utils/StatisticsCollector"; import {WhiskerTestListWithSummary} from "./WhiskerTestListWithSummary"; -import {LocalSearch} from "../search/operators/LocalSearch/LocalSearch"; +import {Randomness} from "../utils/Randomness"; +import {Container} from "../utils/Container"; +import {TestExecutor} from "../testcase/TestExecutor"; +import {WhiskerSearchConfiguration} from "../utils/WhiskerSearchConfiguration"; /** - * A naive approach to generating tests is to simply - * use the chromosome factory and generate completely - * random tests. + * A naive approach to generating tests by always selecting a random event from the set of available events + * determined by the ScratchEventSelector. */ export class RandomTestGenerator extends TestGenerator implements SearchAlgorithm { @@ -47,7 +45,7 @@ export class RandomTestGenerator extends TestGenerator implements SearchAlgorith /** * Saves the number of Generations. */ - private _iterations = 0; + private _iterations: number; /** * Saves the best performing chromosomes seen so far. @@ -59,27 +57,53 @@ export class RandomTestGenerator extends TestGenerator implements SearchAlgorith */ private _archive = new Map(); - async generateTests(project: ScratchProject): Promise { + /** + * Boolean determining if we have reached full test coverage. + */ + protected _fullCoverageReached = false; + + /** + * The minimum number of randomly selected events. + */ + private readonly minSize: number + + /** + * The maximum number of randomly selected events. + */ + private readonly maxSize: number + + constructor(configuration: WhiskerSearchConfiguration, minSize: number, maxSize: number) { + super(configuration); + this.minSize = minSize; + this.maxSize = maxSize; + } + + /** + * Generate tests by randomly sending events to the Scratch-VM. + * After each Iteration the archive is updated with the trace of executed events. + */ + async generateTests(): Promise { + this._iterations = 0; + this._startTime = Date.now(); + StatisticsCollector.getInstance().iterationCount = 0; + StatisticsCollector.getInstance().coveredFitnessFunctionsCount = 0; + StatisticsCollector.getInstance().startTime = Date.now(); this._fitnessFunctions = this.extractCoverageGoals(); StatisticsCollector.getInstance().fitnessFunctionCount = this._fitnessFunctions.size; this._startTime = Date.now(); - let fullCoverageReached = false; - - const chromosomeGenerator = this._config.getChromosomeGenerator(); const stoppingCondition = this._config.getSearchAlgorithmProperties().getStoppingCondition(); + const eventExtractor = this._config.getEventExtractor(); + const randomTestExecutor = new TestExecutor(Container.vmWrapper, eventExtractor, null); + while (!(stoppingCondition.isFinished(this))) { console.log(`Iteration ${this._iterations}, covered goals: ${this._archive.size}/${this._fitnessFunctions.size}`); + const numberOfEvents = Randomness.getInstance().nextInt(this.minSize, this.maxSize + 1); + const randomEventChromosome = new TestChromosome(new List(), undefined, undefined) + await randomTestExecutor.executeRandomEvents(randomEventChromosome, numberOfEvents); + this.updateArchive(randomEventChromosome); this._iterations++; - StatisticsCollector.getInstance().incrementIterationCount(); - const testChromosome = chromosomeGenerator.get(); - await testChromosome.evaluate(); - this.updateArchive(testChromosome); - if (this._archive.size == this._fitnessFunctions.size && !fullCoverageReached) { - fullCoverageReached = true; - StatisticsCollector.getInstance().createdTestsToReachFullCoverage = this._iterations; - StatisticsCollector.getInstance().timeToReachFullCoverage = Date.now() - this._startTime; - } + this.updateStatistics() } const testSuite = await this.getTestSuite(this._tests); this.collectStatistics(testSuite); @@ -106,6 +130,23 @@ export class RandomTestGenerator extends TestGenerator implements SearchAlgorith } } + /** + * Updates the StatisticsCollector on the following points: + * - bestTestSuiteSize + * - iterationCount + * - createdTestsToReachFullCoverage + * - timeToReachFullCoverage + */ + protected updateStatistics(): void { + StatisticsCollector.getInstance().bestTestSuiteSize = this._tests.size(); + StatisticsCollector.getInstance().incrementIterationCount(); + if (this._archive.size == this._fitnessFunctions.size && !this._fullCoverageReached) { + this._fullCoverageReached = true; + StatisticsCollector.getInstance().createdTestsToReachFullCoverage = this._iterations; + StatisticsCollector.getInstance().timeToReachFullCoverage = Date.now() - this._startTime; + } + } + getCurrentSolution(): List { return this._tests; } diff --git a/whisker-main/src/whisker/utils/Randomness.ts b/whisker-main/src/whisker/utils/Randomness.ts index bc287dc6..196af558 100644 --- a/whisker-main/src/whisker/utils/Randomness.ts +++ b/whisker-main/src/whisker/utils/Randomness.ts @@ -84,8 +84,8 @@ export class Randomness { /** * Pick a random integer from a range - * @param min Lower bound of range - * @param max Upper bound of range + * @param min Lower bound of range inclusive + * @param max Upper bound of range exclusive */ public nextInt(min: number, max: number): number { return Math.floor(this.next(min, max)); diff --git a/whisker-main/src/whisker/utils/WhiskerSearchConfiguration.ts b/whisker-main/src/whisker/utils/WhiskerSearchConfiguration.ts index 54a399d7..3123e48f 100644 --- a/whisker-main/src/whisker/utils/WhiskerSearchConfiguration.ts +++ b/whisker-main/src/whisker/utils/WhiskerSearchConfiguration.ts @@ -78,6 +78,7 @@ export class WhiskerSearchConfiguration { this.dict = Preconditions.checkNotUndefined(dict) } + // TODO: Need variation here; we do not always need all properties, e.g MIO has no crossover public getSearchAlgorithmProperties(): SearchAlgorithmProperties { const populationSize = this.dict['population-size'] as number; const chromosomeLength = this.dict['chromosome-length'] as number; @@ -417,7 +418,7 @@ export class WhiskerSearchConfiguration { public getTestGenerator(): TestGenerator { if (this.dict["test-generator"] == "random") { - return new RandomTestGenerator(this); + return new RandomTestGenerator(this, this.dict['minEventSize'], this.dict['maxEventSize']); } else if (this.dict['test-generator'] == 'iterative') { return new IterativeSearchBasedTestGenerator(this); } else if (this.dict['test-generator'] == 'many-objective') { diff --git a/whisker-main/src/whisker/whiskerNet/NetworkChromosome.ts b/whisker-main/src/whisker/whiskerNet/NetworkChromosome.ts index 6f23581d..0ab2e3d4 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkChromosome.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkChromosome.ts @@ -289,7 +289,7 @@ export class NetworkChromosome extends Chromosome { const classificationNode = new ClassificationNode(this.allNodes.size(), event, ActivationFunction.SIGMOID); this.allNodes.add(classificationNode); this.connectOutputNode(classificationNode); - for (const parameter of event.getVariableParameterNames()) { + for (const parameter of event.getSearchParameterNames()) { const regressionNode = new RegressionNode(this.allNodes.size(), event, parameter, ActivationFunction.NONE) this.allNodes.add(regressionNode); this.connectOutputNode(regressionNode); diff --git a/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts b/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts index 74374bf1..bd505c60 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkExecutor.ts @@ -13,7 +13,7 @@ import {InputExtraction} from "./InputExtraction"; import {NeuroevolutionUtil} from "./NeuroevolutionUtil"; import {ScratchEventExtractor} from "../testcase/ScratchEventExtractor"; import {StaticScratchEventExtractor} from "../testcase/StaticScratchEventExtractor"; -import {ParameterTypes} from "../testcase/events/ParameterTypes"; +import {ParameterType} from "../testcase/events/ParameterType"; import Runtime from "scratch-vm/src/engine/runtime" import {WhiskerSearchConfiguration} from "../utils/WhiskerSearchConfiguration"; import {NeuroevolutionScratchEventExtractor} from "../testcase/NeuroevolutionScratchEventExtractor"; @@ -148,9 +148,9 @@ export class NetworkExecutor { // Select the nextEvent, set its parameters and send it to the Scratch-VM const nextEvent: ScratchEvent = this.availableEvents.get(indexOfMaxValue); let args = []; - if (nextEvent.getNumVariableParameters() > 0) { + if (nextEvent.numSearchParameter() > 0) { args = NetworkExecutor.getArgs(nextEvent, network); - nextEvent.setParameter(args, ParameterTypes.REGRESSION); + nextEvent.setParameter(args, ParameterType.REGRESSION); } events.add([nextEvent, args]); this.notify(nextEvent, args); diff --git a/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGenerator.ts b/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGenerator.ts index f06b2f23..162b729f 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGenerator.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGenerator.ts @@ -59,7 +59,7 @@ export abstract class NetworkChromosomeGenerator implements ChromosomeGenerator< */ protected addRegressionNodes(allNodes: List, parameterizedEvents: List, nodeId: number): void { for (const event of parameterizedEvents) { - for (const parameter of event.getVariableParameterNames()) { + for (const parameter of event.getSearchParameterNames()) { // Create the regression Node and add it to the NodeList const regressionNode = new RegressionNode(nodeId++, event, parameter, ActivationFunction.NONE) allNodes.add(regressionNode) diff --git a/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGeneratorFullyConnected.ts b/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGeneratorFullyConnected.ts index ca3c3f4e..1cb24977 100644 --- a/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGeneratorFullyConnected.ts +++ b/whisker-main/src/whisker/whiskerNet/NetworkGenerators/NetworkChromosomeGeneratorFullyConnected.ts @@ -71,7 +71,7 @@ export class NetworkChromosomeGeneratorFullyConnected extends NetworkChromosomeG } // Add regression nodes for each parameter of each parameterized Event - const parameterizedEvents = this._scratchEvents.filter(event => event.getNumVariableParameters() > 0); + const parameterizedEvents = this._scratchEvents.filter(event => event.numSearchParameter() > 0); if (!parameterizedEvents.isEmpty()) { this.addRegressionNodes(allNodes, parameterizedEvents, nodeId); } diff --git a/whisker-main/test/whisker/testcase/EventSelector.test.ts b/whisker-main/test/whisker/testcase/EventSelector.test.ts index 35cd1b8e..5c715343 100644 --- a/whisker-main/test/whisker/testcase/EventSelector.test.ts +++ b/whisker-main/test/whisker/testcase/EventSelector.test.ts @@ -5,10 +5,10 @@ import {RenderedTarget} from "scratch-vm/src/sprites/rendered-target"; class DummyEvent extends ScratchEvent { - getNumVariableParameters(): number { + numSearchParameter(): number { return 0; } - getVariableParameterNames(): string[] { + getSearchParameterNames(): string[] { return []; } stringIdentifier(): string { diff --git a/whisker-main/test/whisker/utils/Randomness.test.ts b/whisker-main/test/whisker/utils/Randomness.test.ts index 105478b7..0d83e53c 100644 --- a/whisker-main/test/whisker/utils/Randomness.test.ts +++ b/whisker-main/test/whisker/utils/Randomness.test.ts @@ -29,7 +29,7 @@ describe("Randomness", () => { const num = random.nextInt(0, 10); expect(num).toBeGreaterThanOrEqual(0); - expect(num).toBeLessThanOrEqual(10); + expect(num).toBeLessThan(10); }); test("Create a float in [0,1]", () => {