diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 48d18a2d5..034e3fe90 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,10 +2,10 @@ name: "CodeQL" on: push: - branches: [ main ] + branches: [ master ] pull_request: # The branches below must be a subset of the branches above - branches: [ main ] + branches: [ master ] schedule: - cron: '41 14 * * 4' diff --git a/.github/workflows/javadoc-publish.yml b/.github/workflows/javadoc-publish.yml index 18b87499b..2d48f3557 100644 --- a/.github/workflows/javadoc-publish.yml +++ b/.github/workflows/javadoc-publish.yml @@ -3,7 +3,7 @@ name: Deploy Javadoc on: push: branches: - - main + - master jobs: publish: diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 9c853a8f2..bc27c8dbf 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -5,9 +5,9 @@ name: Java CI with Maven on: push: - branches: [ main ] + branches: [ master ] pull_request: - branches: [ main ] + branches: [ master ] jobs: build: diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 88d33ec8c..731c5e7f1 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -1,9 +1,9 @@ name: Playwright Tests on: push: - branches: [ main, beta ] + branches: [ master, beta ] pull_request: - branches: [ main, beta ] + branches: [ master, beta ] jobs: test: timeout-minutes: 60 diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index 1cc94ab70..9113a2773 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -2,7 +2,7 @@ name: Build on: push: branches: - - main + - master pull_request: types: [opened, synchronize, reopened] jobs: diff --git a/pom.xml b/pom.xml index 44a6370e8..0bac4f10e 100644 --- a/pom.xml +++ b/pom.xml @@ -36,10 +36,31 @@ https://sonarcloud.io - - + + + jitpack.io + https://jitpack.io + + + + + + com.github.forax + beautiful_logger + 0.10.6 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + 2.15.3 + + + com.microsoft.playwright + playwright + 1.39.0 + org.junit.jupiter @@ -59,15 +80,12 @@ ${junit} test + - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - 2.15.3 - - - com.microsoft.playwright - playwright - 1.39.0 + org.mockito + mockito-core + 4.0.0 + test diff --git a/src/main/java/io/github/mathieusoysal/App.java b/src/main/java/io/github/mathieusoysal/App.java new file mode 100644 index 000000000..75496dc5d --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/App.java @@ -0,0 +1,50 @@ +package io.github.mathieusoysal; + +import java.io.IOException; + +import com.fasterxml.jackson.core.exc.StreamReadException; +import com.fasterxml.jackson.databind.DatabindException; +import com.github.forax.beautifullogger.Logger; + +import io.github.mathieusoysal.exceptions.ApiRequestFailedException; +import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException; +import io.github.mathieusoysal.logement.data.DataCollector; +import io.github.mathieusoysal.logement.data.DataSaver; + +public class App { + private static final Logger LOGGER = Logger.getLogger(); + private static final String MAIL_PROPERTIES_NAME = "MAIL"; + private static final String PASSWORD_PROPERTIES_NAME = "PASSWORD"; + + public static void main(String[] args) + throws StreamReadException, DatabindException, ApiRequestFailedException, IOException, + InterruptedException { + LOGGER.info(() -> "Starting application"); + var logements = DataCollector.getAvailableLogementsWithConnection(getEmail(), getPassword()); + DataSaver.createArchiveLogements(logements); + LOGGER.info(() -> "Application finished"); + } + + private static String getEmail() { + LOGGER.info(() -> "Getting email from environment variables"); + String email = System.getenv(MAIL_PROPERTIES_NAME); + if (email == null) + { + LOGGER.error(() -> "Email not found in environment variables"); + throw new PropertiesNotFoundRuntimeException(MAIL_PROPERTIES_NAME); + } + return email; + } + + private static String getPassword() { + LOGGER.info(() -> "Getting password from environment variables"); + String password = System.getenv(PASSWORD_PROPERTIES_NAME); + if (password == null) + { + LOGGER.error(() -> "Password not found in environment variables"); + throw new PropertiesNotFoundRuntimeException(PASSWORD_PROPERTIES_NAME); + } + return password; + } + +} diff --git a/src/main/java/io/github/mathieusoysal/exceptions/PropertiesNotFoundRuntimeException.java b/src/main/java/io/github/mathieusoysal/exceptions/PropertiesNotFoundRuntimeException.java new file mode 100644 index 000000000..9b87c0188 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/exceptions/PropertiesNotFoundRuntimeException.java @@ -0,0 +1,13 @@ +package io.github.mathieusoysal.exceptions; + +public class PropertiesNotFoundRuntimeException extends RuntimeException { + + public PropertiesNotFoundRuntimeException(String propertiesName) { + super("Properties value " + propertiesName + " not found in environment variables"); + } + + public PropertiesNotFoundRuntimeException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/io/github/mathieusoysal/logement/data/DataCollector.java b/src/main/java/io/github/mathieusoysal/logement/data/DataCollector.java index d2d6e3238..85154a3fd 100644 --- a/src/main/java/io/github/mathieusoysal/logement/data/DataCollector.java +++ b/src/main/java/io/github/mathieusoysal/logement/data/DataCollector.java @@ -3,10 +3,10 @@ import java.io.IOException; import java.nio.file.Paths; import java.util.List; -import java.util.function.BooleanSupplier; import com.fasterxml.jackson.core.exc.StreamReadException; import com.fasterxml.jackson.databind.DatabindException; +import com.github.forax.beautifullogger.Logger; import com.microsoft.playwright.Browser; import com.microsoft.playwright.Browser.NewContextOptions; import com.microsoft.playwright.BrowserContext; @@ -27,6 +27,8 @@ import io.github.mathieusoysal.logement.pojo.Logement; public class DataCollector { + private static final Logger LOGGER = Logger.getLogger(); + private static final String LINK_TO_GET_ALL_LOGEMENTS = "https://trouverunlogement.lescrous.fr/api/fr/search/32"; private static final String BODY_POST_TO_GET_LOGEMENTS = "{\r\n \"idTool\": 32,\r\n \"need_aggregation\": false,\r\n \"page\": 1,\r\n \"pageSize\": 2500,\r\n \"sector\": null,\r\n \"occupationModes\": [],\r\n \"location\": [\r\n {\r\n \"lon\": -9.9079,\r\n \"lat\": 51.7087\r\n },\r\n {\r\n \"lon\": 14.3224,\r\n \"lat\": 40.5721\r\n }\r\n ],\r\n \"residence\": null,\r\n \"precision\": 9,\r\n \"equipment\": [],\r\n \"price\": {\r\n \"min\": 0,\r\n \"max\": 10000000\r\n }\r\n}"; private static final RequestOptions REQUEST_TO_GET_LOGEMENTS = RequestOptions.create() .setMethod("POST") @@ -36,38 +38,53 @@ public class DataCollector { public static List getAvailableLogementsWithoutConnection() throws ApiRequestFailedException, StreamReadException, DatabindException, IOException { List logements; + LOGGER.info(() -> "Creating profil to request logements"); try (Playwright playwright = Playwright.create()) { + LOGGER.info(() -> "profil created"); + LOGGER.info(() -> "Requesting logements from " + LINK_TO_GET_ALL_LOGEMENTS); var respons = playwright.request().newContext() - .head("https://trouverunlogement.lescrous.fr/api/fr/search/32", REQUEST_TO_GET_LOGEMENTS); + .head(LINK_TO_GET_ALL_LOGEMENTS, REQUEST_TO_GET_LOGEMENTS); if (!respons.ok()) throw new ApiRequestFailedException(respons); + LOGGER.info(() -> "Logements received"); logements = Convertor.getLogementsFromBruteJsonString(respons.text()); } + LOGGER.info(() -> "profil closed"); return logements; } public static List getAllLogementsWithoutConnection() throws ApiRequestFailedException, StreamReadException, DatabindException, IOException { + LOGGER.info(() -> "Getting all logements"); List logements; + LOGGER.info(() -> "Creating profil to request logements"); try (Playwright playwright = Playwright.create()) { var respons = playwright.request().newContext() .head("https://trouverunlogement.lescrous.fr/api/fr/search/29", REQUEST_TO_GET_LOGEMENTS); - if (!respons.ok()) + if (!respons.ok()) { + LOGGER.error(() -> "Request failed"); throw new ApiRequestFailedException(respons); + } + LOGGER.info(() -> "Request succeed"); logements = Convertor.getLogementsFromBruteJsonString(respons.text()); } + LOGGER.info(() -> "profil closed"); + LOGGER.info(() -> "All logements received"); return logements; } public static List getAvailableLogementsWithConnection(String email, String password) throws ApiRequestFailedException, StreamReadException, DatabindException, IOException, InterruptedException { + LOGGER.info(() -> "Getting available logements"); + LOGGER.info(() -> "Creating profil to request logements"); Playwright playwright = Playwright.create(); Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions()); BrowserContext context = browser.newContext(new NewContextOptions().setScreenSize(1920, 1080)); Page page = context.newPage(); List logements; try { + context.tracing().start(new Tracing.StartOptions() .setScreenshots(true) .setSnapshots(true) @@ -75,16 +92,22 @@ public static List getAvailableLogementsWithConnection(String email, S goToLoginPage(page); selectLoginOption(playwright, page); connectToTheCrous(email, password, playwright, page); + LOGGER.info(() -> "Going to logements page"); page.getByRole(AriaRole.BUTTON, new Page.GetByRoleOptions().setName("Lancer une recherche")) .click(); page.waitForLoadState(); + LOGGER.info(() -> "Requesting logements from " + LINK_TO_GET_ALL_LOGEMENTS); var respons = page.request() - .head("https://trouverunlogement.lescrous.fr/api/fr/search/32", + .head(LINK_TO_GET_ALL_LOGEMENTS, REQUEST_TO_GET_LOGEMENTS); - if (!respons.ok()) + if (!respons.ok()) { + LOGGER.error(() -> "Request failed"); throw new ApiRequestFailedException(respons); + } + LOGGER.info(() -> "Logements received"); logements = Convertor.getLogementsFromBruteJsonString(respons.text()); } catch (TimeoutError | LoginOptionCantBeSelectedError | CannotBeConnectedError e) { + LOGGER.error("Request failed", e); context.tracing().stop(new Tracing.StopOptions() .setPath(Paths.get("trace.zip"))); throw e; @@ -93,11 +116,13 @@ public static List getAvailableLogementsWithConnection(String email, S context.close(); browser.close(); playwright.close(); + LOGGER.info(() -> "profil closed"); } return logements; } private static void goToLoginPage(Page page) { + LOGGER.info(() -> "Going to login page"); page.navigate("https://trouverunlogement.lescrous.fr/tools/32/search"); page.getByRole(AriaRole.LINK, new Page.GetByRoleOptions().setName("Identification")).click(); page.waitForLoadState(); @@ -106,6 +131,7 @@ private static void goToLoginPage(Page page) { private static void selectLoginOption(Playwright playwright, Page page) { playwright.selectors().setTestIdAttribute("id"); String currentUrl = page.url(); + LOGGER.info(() -> "Selecting login option"); try { page.locator("#boxlogin div").nth(0).click(); if (page.url().equals(currentUrl)) @@ -117,11 +143,14 @@ private static void selectLoginOption(Playwright playwright, Page page) { page.waitForLoadState(LoadState.DOMCONTENTLOADED); waitForUrlChange(currentUrl, page); } catch (TimeoutError e) { + LOGGER.error(() -> "Login option can't be selected"); throw new LoginOptionCantBeSelectedError(e.getMessage(), page.content()); } + LOGGER.info(() -> "Login option selected"); } private static void connectToTheCrous(String email, String password, Playwright playwright, Page page) { + LOGGER.info(() -> "Connecting to the crous"); playwright.selectors().setTestIdAttribute("type"); String currentUrl = page.url(); try { @@ -129,11 +158,14 @@ private static void connectToTheCrous(String email, String password, Playwright waitForPageLoad(page); waitForUrlChange(currentUrl, page); } catch (TimeoutError e) { + LOGGER.error(() -> "Can't connect to the crous"); throw new CannotBeConnectedError(e.getMessage(), page.content()); } + LOGGER.info(() -> "Connected to the crous"); } private static void fillForm(String email, String password, Page page) { + LOGGER.info(() -> "Filling form"); var emailField = page.getByTestId("email"); emailField.hover(); emailField.click(); @@ -141,6 +173,8 @@ private static void fillForm(String email, String password, Page page) { var passwordField = page.getByTestId("password"); passwordField.click(); passwordField.fill(password); + LOGGER.info(() -> "Form filled"); + LOGGER.info(() -> "Submitting form"); passwordField.press("Enter"); } diff --git a/src/main/java/io/github/mathieusoysal/logement/data/DataSaver.java b/src/main/java/io/github/mathieusoysal/logement/data/DataSaver.java new file mode 100644 index 000000000..a04fdc530 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/logement/data/DataSaver.java @@ -0,0 +1,94 @@ +package io.github.mathieusoysal.logement.data; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.time.DateTimeException; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.stream.Stream; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.github.forax.beautifullogger.Logger; + +import io.github.mathieusoysal.logement.pojo.Logement; + +public class DataSaver { + private static final Logger LOGGER = Logger.getLogger(); + + public static File createArchiveFolder() { + LOGGER.info(() -> "getting archive folder"); + File archiveFolder = new File("archive"); + if (!archiveFolder.exists()) { + archiveFolder.mkdir(); + LOGGER.info(() -> "Archive folder created"); + } + + return archiveFolder; + } + + public static File createArchiveLogements(List logements) throws JsonProcessingException { + File archiveFolder = getArchiveFolderForCurrentDate(); + String logementsJson = convertLogementsToJson(logements); + File archiveFile = getArchiveFile(archiveFolder); + writeLogementsDataInsideArchiveFile(logementsJson, archiveFile); + return archiveFile; + } + + private static void writeLogementsDataInsideArchiveFile(String logementsJson, File archiveFile) { + LOGGER.info(() -> "Writing logements to file"); + try (FileWriter fileWriter = new FileWriter(archiveFile)) { + fileWriter.write(logementsJson); + } catch (IOException e) { + LOGGER.error("Error while writing logements to file", e); + throw new RuntimeException("Error while writing logements to file", e); + } + LOGGER.info(() -> "Logements written to file"); + } + + private static File getArchiveFile(File archiveFolder) throws DateTimeException { + LOGGER.info(() -> "Getting archive file"); + String archiveFileName = OffsetDateTime.now().toLocalTime().format(DateTimeFormatter.ofPattern("HH")); + Stream.of(archiveFolder.listFiles()) + .filter(file -> file.getName().equals(archiveFileName)) + .findFirst() + .ifPresent(file -> { + LOGGER.error(() -> "Archive file already exists"); + try { + Files.delete(file.toPath()); + } catch (IOException e) { + LOGGER.error("Error while deleting archive file", e); + e.printStackTrace(); + } + LOGGER.info(() -> "Archive file deleted"); + }); + var archiveFile = new File(archiveFolder, archiveFileName); + LOGGER.info(() -> "Archive file got"); + return archiveFile; + } + + private static String convertLogementsToJson(List logements) throws JsonProcessingException { + LOGGER.info(() -> "Converting logements to json"); + ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); + var result = ow.writeValueAsString(logements); + LOGGER.info(() -> "Logements converted to json"); + return result; + } + + private static File getArchiveFolderForCurrentDate() { + LOGGER.info(() -> "Getting archive folder for current date"); + File archiveFolder = createArchiveFolder(); + String archiveFolderName = OffsetDateTime.now().toLocalDate().toString(); + File archiveFile = new File(archiveFolder, archiveFolderName); + if (!archiveFile.exists()) { + archiveFile.mkdir(); + LOGGER.info(() -> "Archive folder for current date created"); + } + return archiveFile; + } + +} diff --git a/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java b/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java index eebc469d0..ae80eac50 100644 --- a/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java +++ b/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.exc.StreamReadException; import com.fasterxml.jackson.databind.DatabindException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.forax.beautifullogger.Logger; import io.github.mathieusoysal.logement.Address; import io.github.mathieusoysal.logement.BedKind; @@ -16,24 +17,32 @@ import io.github.mathieusoysal.logement.TransportUnitOfMeasure; public class Convertor { + private static final Logger LOGGER = Logger.getLogger(); private Convertor() { } static List getItemsFromJsonFile(File file) throws StreamReadException, DatabindException, IOException { + LOGGER.info(() -> "Reading json file for convertion to java object"); ObjectMapper objectMapper = new ObjectMapper(); Input results = objectMapper.readValue(file, Input.class); + LOGGER.info(() -> "Json file converted to java object"); return results.getResults().getItems(); } static List getItemsFromJsonString(String json) throws StreamReadException, DatabindException, IOException { + LOGGER.info(() -> "Reading json string for convertion to java object"); ObjectMapper objectMapper = new ObjectMapper(); Input results = objectMapper.readValue(json, Input.class); + LOGGER.info(() -> "Json string converted to java object"); return results.getResults().getItems(); } static List convertItemsToLogements(List items) { - return items.stream().map(Convertor::convertItemToLogement).toList(); + LOGGER.info(() -> "Converting items to logements"); + var result = items.stream().map(Convertor::convertItemToLogement).toList(); + LOGGER.info(() -> "Items converted to logements"); + return result; } public static List getLogementsFromBruteJsonFile(File file) diff --git a/src/test/java/io/github/mathieusoysal/logement/data/DataSaverTest.java b/src/test/java/io/github/mathieusoysal/logement/data/DataSaverTest.java new file mode 100644 index 000000000..86a3352cb --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/logement/data/DataSaverTest.java @@ -0,0 +1,43 @@ +package io.github.mathieusoysal.logement.data; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import io.github.mathieusoysal.exceptions.ApiRequestFailedException; +import io.github.mathieusoysal.logement.pojo.Logement; + +class DataSaverTest { + + @AfterEach + void tearDown() { + File archiveFolder = new File("archive"); + if (archiveFolder.exists()) + archiveFolder.delete(); + } + + @Test + void testCreateArchiveFolder() { + File archiveFolder = DataSaver.createArchiveFolder(); + + assertTrue(archiveFolder.exists()); + assertTrue(archiveFolder.isDirectory()); + assertEquals("archive", archiveFolder.getName()); + archiveFolder.delete(); + } + + @Test + void testCreateArchiveLogements() throws ApiRequestFailedException, IOException { + List logements = DataCollector.getAllLogementsWithoutConnection().stream().limit(2).toList(); + var file = assertDoesNotThrow(() -> DataSaver.createArchiveLogements(logements)); + file.delete(); + } + +}