Merge pull request #19 from MathieuSoysal/12-convertor-logement-objec…

…t-json

12 convertor logement object json

.github/workflows/codeql-analysis.yml

@@ -2,10 +2,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ main ]
+    branches: [ master ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ main ]
+    branches: [ master ]
   schedule:
     - cron: '41 14 * * 4'
 

.github/workflows/javadoc-publish.yml

@@ -3,7 +3,7 @@ name: Deploy Javadoc
 on:
   push:
     branches:
-      - main
+      - master
 
 jobs:
   publish:

.github/workflows/maven.yml

@@ -5,9 +5,9 @@ name: Java CI with Maven
 
 on:
   push:
-    branches: [ main ]
+    branches: [ master ]
   pull_request:
-    branches: [ main ]
+    branches: [ master ]
 
 jobs:
   build:

.github/workflows/playwright.yml

@@ -1,9 +1,9 @@
 name: Playwright Tests
 on:
   push:
-    branches: [ main, beta ]
+    branches: [ master, beta ]
   pull_request:
-    branches: [ main, beta ]
+    branches: [ master, beta ]
 jobs:
   test:
     timeout-minutes: 60

.github/workflows/sonar.yml

@@ -2,7 +2,7 @@ name: Build
 on:
   push:
     branches:
-      - main
+      - master
   pull_request:
     types: [opened, synchronize, reopened]
 jobs:

pom.xml

@@ -36,10 +36,31 @@
         <sonar.host.url>https://sonarcloud.io</sonar.host.url>
     </properties>
 
-    <dependencies>
-        <!-- Dependencies -->
+    <repositories>
+        <repository>
+            <id>jitpack.io</id>
+            <url>https://jitpack.io</url>
+        </repository>
+    </repositories>
 
 
+    <dependencies>
+        <!-- Dependencies -->
+        <dependency>
+            <groupId>com.github.forax</groupId>
+            <artifactId>beautiful_logger</artifactId>
+            <version>0.10.6</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-xml</artifactId>
+            <version>2.15.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.playwright</groupId>
+            <artifactId>playwright</artifactId>
+            <version>1.39.0</version>
+        </dependency>
         <!-- Testing dependencies-->
         <dependency>
             <groupId>org.junit.jupiter</groupId>
@@ -59,15 +80,12 @@
             <version>${junit}</version>
             <scope>test</scope>
         </dependency>
+        <!-- Add mockito -->
         <dependency>
-            <groupId>com.fasterxml.jackson.dataformat</groupId>
-            <artifactId>jackson-dataformat-xml</artifactId>
-            <version>2.15.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.microsoft.playwright</groupId>
-            <artifactId>playwright</artifactId>
-            <version>1.39.0</version>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>4.0.0</version>
+            <scope>test</scope>
         </dependency>
     </dependencies>
 

src/main/java/io/github/mathieusoysal/App.java

@@ -0,0 +1,50 @@
+package io.github.mathieusoysal;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.exc.StreamReadException;
+import com.fasterxml.jackson.databind.DatabindException;
+import com.github.forax.beautifullogger.Logger;
+
+import io.github.mathieusoysal.exceptions.ApiRequestFailedException;
+import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException;
+import io.github.mathieusoysal.logement.data.DataCollector;
+import io.github.mathieusoysal.logement.data.DataSaver;
+
+public class App {
+    private static final Logger LOGGER = Logger.getLogger();
+    private static final String MAIL_PROPERTIES_NAME = "MAIL";
+    private static final String PASSWORD_PROPERTIES_NAME = "PASSWORD";
+
+    public static void main(String[] args)
+            throws StreamReadException, DatabindException, ApiRequestFailedException, IOException,
+            InterruptedException {
+        LOGGER.info(() -> "Starting application");
+        var logements = DataCollector.getAvailableLogementsWithConnection(getEmail(), getPassword());
+        DataSaver.createArchiveLogements(logements);
+        LOGGER.info(() -> "Application finished");
+    }
+
+    private static String getEmail() {
+        LOGGER.info(() -> "Getting email from environment variables");
+        String email = System.getenv(MAIL_PROPERTIES_NAME);
+        if (email == null)
+        {
+            LOGGER.error(() -> "Email not found in environment variables");
+            throw new PropertiesNotFoundRuntimeException(MAIL_PROPERTIES_NAME);
+        }
+        return email;
+    }
+
+    private static String getPassword() {
+        LOGGER.info(() -> "Getting password from environment variables");
+        String password = System.getenv(PASSWORD_PROPERTIES_NAME);
+        if (password == null)
+        {
+            LOGGER.error(() -> "Password not found in environment variables");
+            throw new PropertiesNotFoundRuntimeException(PASSWORD_PROPERTIES_NAME);
+        }
+        return password;
+    }
+
+}

src/main/java/io/github/mathieusoysal/exceptions/PropertiesNotFoundRuntimeException.java

@@ -0,0 +1,13 @@
+package io.github.mathieusoysal.exceptions;
+
+public class PropertiesNotFoundRuntimeException extends RuntimeException {
+
+    public PropertiesNotFoundRuntimeException(String propertiesName) {
+        super("Properties value " + propertiesName + " not found in environment variables");
+    }
+
+    public PropertiesNotFoundRuntimeException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+}

src/main/java/io/github/mathieusoysal/logement/data/DataCollector.java

@@ -3,10 +3,10 @@
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.List;
-import java.util.function.BooleanSupplier;
 
 import com.fasterxml.jackson.core.exc.StreamReadException;
 import com.fasterxml.jackson.databind.DatabindException;
+import com.github.forax.beautifullogger.Logger;
 import com.microsoft.playwright.Browser;
 import com.microsoft.playwright.Browser.NewContextOptions;
 import com.microsoft.playwright.BrowserContext;
@@ -27,6 +27,8 @@
 import io.github.mathieusoysal.logement.pojo.Logement;
 
 public class DataCollector {
+    private static final Logger LOGGER = Logger.getLogger();
+    private static final String LINK_TO_GET_ALL_LOGEMENTS = "https://trouverunlogement.lescrous.fr/api/fr/search/32";
     private static final String BODY_POST_TO_GET_LOGEMENTS = "{\r\n    \"idTool\": 32,\r\n    \"need_aggregation\": false,\r\n    \"page\": 1,\r\n    \"pageSize\": 2500,\r\n    \"sector\": null,\r\n    \"occupationModes\": [],\r\n    \"location\": [\r\n        {\r\n            \"lon\": -9.9079,\r\n            \"lat\": 51.7087\r\n        },\r\n        {\r\n            \"lon\": 14.3224,\r\n            \"lat\": 40.5721\r\n        }\r\n    ],\r\n    \"residence\": null,\r\n    \"precision\": 9,\r\n    \"equipment\": [],\r\n    \"price\": {\r\n        \"min\": 0,\r\n        \"max\": 10000000\r\n    }\r\n}";
     private static final RequestOptions REQUEST_TO_GET_LOGEMENTS = RequestOptions.create()
             .setMethod("POST")
@@ -36,55 +38,76 @@ public class DataCollector {
     public static List<Logement> getAvailableLogementsWithoutConnection()
             throws ApiRequestFailedException, StreamReadException, DatabindException, IOException {
         List<Logement> logements;
+        LOGGER.info(() -> "Creating profil to request logements");
         try (Playwright playwright = Playwright.create()) {
+            LOGGER.info(() -> "profil created");
+            LOGGER.info(() -> "Requesting logements from " + LINK_TO_GET_ALL_LOGEMENTS);
             var respons = playwright.request().newContext()
-                    .head("https://trouverunlogement.lescrous.fr/api/fr/search/32", REQUEST_TO_GET_LOGEMENTS);
+                    .head(LINK_TO_GET_ALL_LOGEMENTS, REQUEST_TO_GET_LOGEMENTS);
             if (!respons.ok())
                 throw new ApiRequestFailedException(respons);
+            LOGGER.info(() -> "Logements received");
             logements = Convertor.getLogementsFromBruteJsonString(respons.text());
         }
+        LOGGER.info(() -> "profil closed");
         return logements;
     }
 
     public static List<Logement> getAllLogementsWithoutConnection()
             throws ApiRequestFailedException, StreamReadException, DatabindException, IOException {
+        LOGGER.info(() -> "Getting all logements");
         List<Logement> logements;
+        LOGGER.info(() -> "Creating profil to request logements");
         try (Playwright playwright = Playwright.create()) {
             var respons = playwright.request().newContext()
                     .head("https://trouverunlogement.lescrous.fr/api/fr/search/29", REQUEST_TO_GET_LOGEMENTS);
-            if (!respons.ok())
+            if (!respons.ok()) {
+                LOGGER.error(() -> "Request failed");
                 throw new ApiRequestFailedException(respons);
+            }
+            LOGGER.info(() -> "Request succeed");
             logements = Convertor.getLogementsFromBruteJsonString(respons.text());
         }
+        LOGGER.info(() -> "profil closed");
+        LOGGER.info(() -> "All logements received");
         return logements;
     }
 
     public static List<Logement> getAvailableLogementsWithConnection(String email, String password)
             throws ApiRequestFailedException, StreamReadException, DatabindException, IOException,
             InterruptedException {
+        LOGGER.info(() -> "Getting available logements");
+        LOGGER.info(() -> "Creating profil to request logements");
         Playwright playwright = Playwright.create();
         Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions());
         BrowserContext context = browser.newContext(new NewContextOptions().setScreenSize(1920, 1080));
         Page page = context.newPage();
         List<Logement> logements;
         try {
+
             context.tracing().start(new Tracing.StartOptions()
                     .setScreenshots(true)
                     .setSnapshots(true)
                     .setSources(true));
             goToLoginPage(page);
             selectLoginOption(playwright, page);
             connectToTheCrous(email, password, playwright, page);
+            LOGGER.info(() -> "Going to logements page");
             page.getByRole(AriaRole.BUTTON, new Page.GetByRoleOptions().setName("Lancer une recherche"))
                     .click();
             page.waitForLoadState();
+            LOGGER.info(() -> "Requesting logements from " + LINK_TO_GET_ALL_LOGEMENTS);
             var respons = page.request()
-                    .head("https://trouverunlogement.lescrous.fr/api/fr/search/32",
+                    .head(LINK_TO_GET_ALL_LOGEMENTS,
                             REQUEST_TO_GET_LOGEMENTS);
-            if (!respons.ok())
+            if (!respons.ok()) {
+                LOGGER.error(() -> "Request failed");
                 throw new ApiRequestFailedException(respons);
+            }
+            LOGGER.info(() -> "Logements received");
             logements = Convertor.getLogementsFromBruteJsonString(respons.text());
         } catch (TimeoutError | LoginOptionCantBeSelectedError | CannotBeConnectedError e) {
+            LOGGER.error("Request failed", e);
             context.tracing().stop(new Tracing.StopOptions()
                     .setPath(Paths.get("trace.zip")));
             throw e;
@@ -93,11 +116,13 @@ public static List<Logement> getAvailableLogementsWithConnection(String email, S
             context.close();
             browser.close();
             playwright.close();
+            LOGGER.info(() -> "profil closed");
         }
         return logements;
     }
 
     private static void goToLoginPage(Page page) {
+        LOGGER.info(() -> "Going to login page");
         page.navigate("https://trouverunlogement.lescrous.fr/tools/32/search");
         page.getByRole(AriaRole.LINK, new Page.GetByRoleOptions().setName("Identification")).click();
         page.waitForLoadState();
@@ -106,6 +131,7 @@ private static void goToLoginPage(Page page) {
     private static void selectLoginOption(Playwright playwright, Page page) {
         playwright.selectors().setTestIdAttribute("id");
         String currentUrl = page.url();
+        LOGGER.info(() -> "Selecting login option");
         try {
             page.locator("#boxlogin div").nth(0).click();
             if (page.url().equals(currentUrl))
@@ -117,30 +143,38 @@ private static void selectLoginOption(Playwright playwright, Page page) {
             page.waitForLoadState(LoadState.DOMCONTENTLOADED);
             waitForUrlChange(currentUrl, page);
         } catch (TimeoutError e) {
+            LOGGER.error(() -> "Login option can't be selected");
             throw new LoginOptionCantBeSelectedError(e.getMessage(), page.content());
         }
+        LOGGER.info(() -> "Login option selected");
     }
 
     private static void connectToTheCrous(String email, String password, Playwright playwright, Page page) {
+        LOGGER.info(() -> "Connecting to the crous");
         playwright.selectors().setTestIdAttribute("type");
         String currentUrl = page.url();
         try {
             fillForm(email, password, page);
             waitForPageLoad(page);
             waitForUrlChange(currentUrl, page);
         } catch (TimeoutError e) {
+            LOGGER.error(() -> "Can't connect to the crous");
             throw new CannotBeConnectedError(e.getMessage(), page.content());
         }
+        LOGGER.info(() -> "Connected to the crous");
     }
 
     private static void fillForm(String email, String password, Page page) {
+        LOGGER.info(() -> "Filling form");
         var emailField = page.getByTestId("email");
         emailField.hover();
         emailField.click();
         emailField.fill(email);
         var passwordField = page.getByTestId("password");
         passwordField.click();
         passwordField.fill(password);
+        LOGGER.info(() -> "Form filled");
+        LOGGER.info(() -> "Submitting form");
         passwordField.press("Enter");
     }