Merge pull request #457 from usethesource/chore/ci-faster-tests

Improving the UI test runtime

.github/workflows/build.yaml

@@ -14,6 +14,37 @@ permissions:
   contents: write
 
 jobs:
+  normal-tests:
+    runs-on: buildjet-2vcpu-ubuntu-2204
+    steps:
+      - uses: actions/checkout@v4
+      - name: Assure different pom cache hit
+        run: echo "  " >> rascal-lsp/pom.xml
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'temurin'
+          cache: 'maven'
+
+      - name: setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+          cache: 'npm'
+          cache-dependency-path: rascal-vscode-extension/package-lock.json
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Run java tests
+        working-directory: ./rascal-lsp
+        run: mvn -B '-Drascal.compile.skip' '-Drascal.tutor.skip' test -DargLine="-Xmx4G"
+
+      - name: VS Code tests
+        working-directory: ./rascal-vscode-extension
+        run: |
+          npm ci
+          npm run compile-tests
+          npm run normalTest
+      
   ui-test:
     strategy:
       matrix:
@@ -24,6 +55,8 @@ jobs:
     runs-on:  ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - name: Assure different pom cache hit
+        run: echo " " >> rascal-lsp/pom.xml
       - uses: actions/setup-java@v4
         with:
           java-version: 11
@@ -40,7 +73,7 @@ jobs:
 
       - name: Package & compile rascal-lsp
         working-directory: ./rascal-lsp
-        run: mvn -B '-Drascal.compile.skip' '-Drascal.tutor.skip' clean package
+        run: mvn -B '-Drascal.compile.skip' '-Drascal.tutor.skip' -DskipTests clean package
 
       - name: Package & compile extension 
         working-directory: ./rascal-vscode-extension
@@ -60,7 +93,7 @@ jobs:
         working-directory: ./rascal-vscode-extension
         env:
           DELAY_FACTOR: 10
-          _JAVA_OPTIONS: '-Xmx2G'
+          _JAVA_OPTIONS: '-Xmx5G'
         run: npx extest setup-and-run out/test/vscode-suite/*.test.js --storage uitests
 
       - name: "UI test (mac)"
@@ -69,7 +102,7 @@ jobs:
         working-directory: ./rascal-vscode-extension
         env:
           DELAY_FACTOR: 15
-          _JAVA_OPTIONS: '-Xmx4G'
+          _JAVA_OPTIONS: '-Xmx5G'
         run: npx extest setup-and-run out/test/vscode-suite/*.test.js --storage uitests
 
       - name: "UI test (ubuntu)"
@@ -95,7 +128,7 @@ jobs:
         run: rm -rf ./rascal-vscode-extension/uitests/screenshots
 
   build:
-    needs: [ui-test]
+    needs: [ui-test, normal-tests]
     runs-on: buildjet-2vcpu-ubuntu-2204
     steps:
       - uses: actions/checkout@v4
@@ -115,7 +148,7 @@ jobs:
 
       - name: Package & compile rascal-lsp
         working-directory: ./rascal-lsp
-        run: mvn -B clean verify
+        run: mvn -B clean verify -DskipTests
         env:
           MAVEN_OPTS: "-Xmx6G"
 
@@ -131,9 +164,6 @@ jobs:
           npm run license-check
           npm run esbuild
           npm run lint 
-          npm run compile-tests
-          npm run normalTest
-          # disable vs-code test npm run test
 
       - name: package extension
         working-directory: rascal-vscode-extension

rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts

@@ -48,7 +48,7 @@ describe('DSL', function () {
         await repl.start();
         await repl.execute("import demo::lang::pico::LanguageServer;");
         repl.execute("main();"); // we don't wait, be cause we might miss pico loading window
-        const ide = new IDEOperations(browser, bench);
+        const ide = new IDEOperations(browser);
         const isPicoLoading = ide.statusContains("Pico");
         await driver.wait(isPicoLoading, Delays.slow, "Pico DSL should start loading");
         await repl.terminate();
@@ -62,11 +62,11 @@ describe('DSL', function () {
         driver = browser.driver;
         bench = new Workbench();
         await ignoreFails(browser.waitForWorkbench());
-        ide = new IDEOperations(browser, bench);
+        ide = new IDEOperations(browser);
         await ide.load();
         await loadPico();
         picoFileBackup = await fs.readFile(TestWorkspace.picoFile);
-        ide = new IDEOperations(browser, bench);
+        ide = new IDEOperations(browser);
         await ide.load();
     });
 
@@ -86,7 +86,7 @@ describe('DSL', function () {
         await ide.hasSyntaxHighlighting(editor);
         try {
             await editor.setTextAtLine(10, "b := ;");
-            await ide.hasErrorSquiggly(editor, 15_000);
+            await ide.hasErrorSquiggly(editor, Delays.slow);
         } finally {
             await ide.revertOpenChanges();
         }
@@ -97,7 +97,7 @@ describe('DSL', function () {
         await ide.hasSyntaxHighlighting(editor);
         try {
             await editor.setTextAtLine(10, "b := ;");
-            await ide.hasErrorSquiggly(editor, 15_000);
+            await ide.hasErrorSquiggly(editor, Delays.slow);
         } finally {
             await ide.revertOpenChanges();
         }
@@ -113,7 +113,7 @@ describe('DSL', function () {
         const editor = await ide.openModule(TestWorkspace.picoFile);
         try {
             await editor.setTextAtLine(10, "bzzz := 3;");
-            await ide.hasErrorSquiggly(editor, 15_000);
+            await ide.hasErrorSquiggly(editor, Delays.slow);
         } finally {
             await ide.revertOpenChanges();
         }
@@ -125,8 +125,8 @@ describe('DSL', function () {
         try {
             await editor.setTextAtLine(10, "bzzz := 3;");
             await editor.save();
-            await ide.hasWarningSquiggly(editor, 15_000);
-            await ide.hasErrorSquiggly(editor, 15_000);
+            await ide.hasWarningSquiggly(editor, Delays.slow);
+            await ide.hasErrorSquiggly(editor, Delays.slow);
         } finally {
             await editor.setTextAtLine(10, line10);
             await editor.save();
@@ -138,14 +138,21 @@ describe('DSL', function () {
         await ide.triggerTypeChecker(editor, {checkName: "Pico check"});
         await editor.selectText("x", 2);
         await bench.executeCommand("Go to Definition");
-        await driver.wait(async ()=> (await editor.getCoordinates())[0] === 3, 15_000, "Cursor should have moved to line 3");
+        await driver.wait(async ()=> (await editor.getCoordinates())[0] === 3, Delays.slow, "Cursor should have moved to line 3");
     });
 
+    function assertLineBecomes(editor: TextEditor, lineNumber: number, lineContents: string, msg: string, wait = Delays.verySlow) : Promise<boolean> {
+        return driver.wait(async () => {
+            const currentContent = (await editor.getTextAtLine(lineNumber)).trim();
+            return currentContent === lineContents;
+        }, wait, msg, 100);
+    }
+
     it("code lens works", async () => {
         const editor = await ide.openModule(TestWorkspace.picoFile);
-        const lens = await driver.wait(async () => editor.getCodeLens("Rename variables a to b."), 10_000, "Rename lens should be available");
+        const lens = await driver.wait(() => editor.getCodeLens("Rename variables a to b."), Delays.verySlow, "Rename lens should be available");
         await lens!.click();
-        await driver.wait(async () => (await editor.getTextAtLine(9)).trim() === "b := 2;", 20_000, "a variable should be changed to b");
+        await assertLineBecomes(editor, 9, "b := 2;", "a variable should be changed to b");
     });
 
     it("quick fix works", async() => {
@@ -156,17 +163,15 @@ describe('DSL', function () {
 
         const inputarea = await editor.findElement(By.className('inputarea'));
         await inputarea.sendKeys(Key.chord(TextEditor.ctlKey, "."));
-        await new Promise((res) => setTimeout(res, Delays.normal));
 
         // finds an open menu with the right item in it (Change to a) and then select
         // the parent that handles UI events like click() and sendKeys()
-        const menuContainer = await driver.wait(() => editor.findElement(By.xpath("//div[contains(@class, 'focused') and contains(@class, 'action')]/span[contains(text(), 'Change to a')]//ancestor::*[contains(@class, 'monaco-list')]")), Delays.normal, "The Change to a option should be available and focussed by default");
+        const menuContainer = await ide.hasElement(editor, By.xpath("//div[contains(@class, 'focused') and contains(@class, 'action')]/span[contains(text(), 'Change to a')]//ancestor::*[contains(@class, 'monaco-list')]"), Delays.normal,"The Change to a option should be available and focussed by default");
 
         // menu container works a bit strangely, it ask the focus to keep track of it,
         // and manages clicks and menus on the highest level (not per item).
         await menuContainer.sendKeys(Key.RETURN);
-
-        await driver.wait(async () => (await editor.getTextAtLine(9)).trim() === "a := 2;", Delays.extremelySlow, "a variable should be changed back to a");
+        await assertLineBecomes(editor, 9, "a := 2;", "a variable should be changed back to a", Delays.extremelySlow);
     });
 
 });

rascal-vscode-extension/src/test/vscode-suite/ide.test.ts

@@ -50,7 +50,7 @@ describe('IDE', function () {
         driver = browser.driver;
         bench = new Workbench();
         await browser.waitForWorkbench();
-        ide = new IDEOperations(browser, bench);
+        ide = new IDEOperations(browser);
         await ide.load();
         // trigger rascal type checker to be sure
         for (const f of protectFiles) {
@@ -80,20 +80,20 @@ describe('IDE', function () {
             const checkRascalStatus = ide.statusContains("Loading Rascal");
 
             for (let tries = 0; tries < 10 && !statusBarSeen; tries++) {
-                await sleep(delay / 80);
                 if (await checkRascalStatus()) {
                     statusBarSeen = true;
                     break;
                 }
+                await sleep(delay / 80);
             }
 
             if (statusBarSeen) {
                 console.log("Waiting for startup of rascal core");
                 for (let tries = 0; tries < 70; tries++) {
-                    await sleep(delay / 80);
                     if (!await checkRascalStatus()) {
                         return;
                     }
+                    await sleep(delay / 80);
                 }
                 console.log("*** warning, loading rascal-core is still running, but we will continue anyway");
             }
@@ -149,12 +149,13 @@ describe('IDE', function () {
         const editor = await ide.openModule(TestWorkspace.mainFile);
         await editor.moveCursor(1,1);
         const explorer = await (await bench.getActivityBar().getViewControl("Explorer"))!.openView();
-        await sleep(Delays.normal);
-        const outline = await explorer.getContent().getSection("Outline") as ViewSection;
+        const outline = await driver.wait(() => explorer.getContent().getSection("Outline"), Delays.normal) as ViewSection;
         await outline.expand();
         const mainItem = await driver.wait(async() => ignoreFails(outline.findItem("main()", 0)), Delays.slow, "Main function should show in the outline");
-        await driver.actions().doubleClick(mainItem!).perform();
-        await driver.wait(async ()=> (await editor.getCoordinates())[0] === 5, Delays.normal, "Cursor should have moved to line that contains the println function");
+        await driver.wait(async () => {
+            await driver.actions().doubleClick(mainItem!).perform();
+            return (await editor.getCoordinates())[0] === 5;
+        }, Delays.normal, "Cursor should have moved to line that contains the println function");
     });
 
     it ("rename works", async() => {

rascal-vscode-extension/src/test/vscode-suite/repl.test.ts

@@ -43,7 +43,7 @@ describe('REPL', function () {
         browser = VSBrowser.instance;
         driver = browser.driver;
         bench = new Workbench();
-        ide = new IDEOperations(browser, bench);
+        ide = new IDEOperations(browser);
         await ide.load();
         await ide.cleanup();
         await browser.waitForWorkbench();
@@ -83,8 +83,8 @@ describe('REPL', function () {
     it("edit call module via repl", async() => {
         const repl = new RascalREPL(bench, driver);
         await repl.start();
-        await repl.execute(":edit demo::lang::pico::LanguageServer");
+        await repl.execute(":edit demo::lang::pico::LanguageServer", true, Delays.extremelySlow);
 
-        await driver.wait(async () => await (await bench.getEditorView().getActiveTab())?.getTitle() === "LanguageServer.rsc", Delays.normal, "LanguageServer should be opened");
+        await driver.wait(async () => await (await bench.getEditorView().getActiveTab())?.getTitle() === "LanguageServer.rsc", Delays.slow, "LanguageServer should be opened");
     });
 });