From b7758fc144c0551b52928a187d725bca65629f71 Mon Sep 17 00:00:00 2001 From: marko-bekhta Date: Tue, 9 Jan 2024 16:10:54 +0100 Subject: [PATCH 1/8] Keep git clones on disk and update them instead of re-cloning on each reindexing --- .../search/app/fetching/FetchingService.java | 111 ++++++++----- .../search/app/quarkusio/QuarkusIO.java | 10 +- .../search/app/util/CloseableDirectory.java | 7 + .../search/app/util/GitCloneDirectory.java | 116 ++++++++++++- .../app/fetching/FetchingServiceTest.java | 156 ++++++++++++++++++ 5 files changed, 346 insertions(+), 54 deletions(-) diff --git a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java index d3e5041c..7959a41b 100644 --- a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java +++ b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java @@ -5,12 +5,17 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -24,16 +29,13 @@ import io.quarkus.logging.Log; import io.quarkus.runtime.LaunchMode; +import org.hibernate.search.util.common.impl.Closer; import org.hibernate.search.util.common.impl.SuppressingCloser; -import org.apache.commons.io.function.IOBiFunction; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.jboss.logging.Logger; - @ApplicationScoped public class FetchingService { - private static final Logger log = Logger.getLogger(FetchingService.class); + private static final Branches MAIN = new Branches(QuarkusIO.SOURCE_BRANCH, QuarkusIO.PAGES_BRANCH); + private static final Branches LOCALIZED = new Branches(QuarkusIO.LOCALIZED_SOURCE_BRANCH, QuarkusIO.LOCALIZED_PAGES_BRANCH); @Inject FetchingConfig fetchingConfig; @@ -41,6 +43,9 @@ public class FetchingService { @Inject QuarkusIOConfig quarkusIOConfig; + private final Map repositories = new HashMap<>(); + private final Set closeableDirectories = new HashSet<>(); + public QuarkusIO fetchQuarkusIo() { CompletableFuture main = null; Map> localized = new LinkedHashMap<>(); @@ -48,14 +53,13 @@ public QuarkusIO fetchQuarkusIo() { ? CloseableDirectory.temp("quarkus.io-unzipped") : null; SimpleExecutor executor = new SimpleExecutor(fetchingConfig.parallelism())) { - main = executor.submit(() -> fetchQuarkusIoSite("quarkus.io", quarkusIOConfig.gitUri(), - QuarkusIO.SOURCE_BRANCH, QuarkusIO.PAGES_BRANCH, unzipDir)); + main = executor.submit(() -> fetchQuarkusIoSite("quarkus.io", quarkusIOConfig.gitUri(), MAIN, unzipDir)); for (Map.Entry entry : sortMap(quarkusIOConfig.localized()).entrySet()) { var language = entry.getKey(); var config = entry.getValue(); localized.put(language, - executor.submit(() -> fetchQuarkusIoSite(language.code + ".quarkus.io", config.gitUri(), - QuarkusIO.LOCALIZED_SOURCE_BRANCH, QuarkusIO.LOCALIZED_PAGES_BRANCH, unzipDir))); + executor.submit( + () -> fetchQuarkusIoSite(language.code + ".quarkus.io", config.gitUri(), LOCALIZED, unzipDir))); } executor.waitForSuccessOrThrow(fetchingConfig.timeout()); // If we get here, all tasks succeeded. @@ -69,10 +73,25 @@ public QuarkusIO fetchQuarkusIo() { } } - private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, String sourceBranch, String pagesBranch, + private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branches branches, CloseableDirectory unzipDir) { + // We only want to clean up a clone directory if it is a clone from a remote repository or an unzipped one. + // If we got a local repository we want to keep it as is and no cloning is needed. + // If it was a local unzipped version -- the "cloneDir" will be removed sooner than we are done with indexing, + // so we want to "clone" that directory, and then it'll be removed as if a remote repository clone. + boolean requiresCloning = !isFile(gitUri) || isZip(gitUri); + URI requestedGitUri = gitUri; + CloseableDirectory cloneDir = null; try { + GitCloneDirectory.GitDirectoryDetails repository = repositories.get(gitUri); + if (LaunchMode.DEVELOPMENT.equals(LaunchMode.current()) && isZip(gitUri)) { + if (repository != null) { + // We are working with a zip file, so we have nothing to refresh as there's no actual remote available; + // just return the same repository without any changes: + return repository.open(); + } + Log.warnf("Unzipping '%s': this application is most likely indexing only a sample of %s." + " See README to index the full website.", gitUri, siteName); @@ -80,12 +99,28 @@ private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, String unzip(Path.of(gitUri), unzippedPath); gitUri = unzippedPath.toUri(); // Fall-through and clone the directory. - // While technically unnecessary (we could use the unzipped directory directly), - // this cloning ensures we run the same code in dev mode as in prod. + // This cloning is as if we are cloning a remote repository. + } else { + if (repository != null) { + // It's not a zip, so it may be either a local directory or a remote repository + // let's pull the changes from it as it shouldn't cause any exceptions, as a remote is actually out there. + return repository.pull(branches); + } } - return gitClone(siteName, gitUri, List.of(sourceBranch, pagesBranch), - (git, directory) -> new GitCloneDirectory(git, directory, pagesBranch)); + + cloneDir = requiresCloning + ? CloseableDirectory.temp(siteName) + : CloseableDirectory.of(Paths.get(gitUri)); + + closeableDirectories.add(cloneDir); + + repository = new GitCloneDirectory.GitDirectoryDetails(cloneDir.path(), branches.pages()); + repositories.put(requestedGitUri, repository); + + // If we have a local repository -- just open it, clone it otherwise: + return requiresCloning ? repository.clone(gitUri, branches) : repository.open(); } catch (RuntimeException | IOException e) { + new SuppressingCloser(e).push(cloneDir); throw new IllegalStateException("Failed to fetch '%s': %s".formatted(siteName, e.getMessage()), e); } } @@ -98,35 +133,31 @@ private Map sortMap(Map T gitClone(String name, URI gitUri, List branches, - IOBiFunction function) { - Log.infof("Cloning '%s' from '%s'.", name, gitUri); - CloseableDirectory directory = null; - Git git = null; - try { - directory = CloseableDirectory.temp(name); - git = Git.cloneRepository() - .setURI(gitUri.toString()) - .setDirectory(directory.path().toFile()) - .setDepth(1) - .setNoTags() - .setBranch(branches.get(0)) - .setBranchesToClone(branches.stream().map(b -> "refs/heads/" + b).toList()) - .setProgressMonitor(LoggerProgressMonitor.create(log, "Cloning " + name + ": ")) - // Unfortunately sparse checkouts are not supported: https://www.eclipse.org/forums/index.php/t/1094825/ - .call(); - return function.apply(git, directory); - } catch (RuntimeException | IOException | GitAPIException e) { - new SuppressingCloser(e) - .push(git) - .push(directory); + @PreDestroy + public void cleanupTemporaryFolders() { + try (Closer closer = new Closer<>()) { + closer.pushAll(CloseableDirectory::close, closeableDirectories); + } catch (Exception e) { throw new IllegalStateException( - "Failed to clone git repository '%s' from '%s': %s".formatted(name, gitUri, e.getMessage()), e); + "Failed to close directories '%s': %s".formatted(closeableDirectories, e.getMessage()), e); + } + } + + public record Branches(String sources, String pages) { + public List asRefList() { + return List.of("refs/heads/" + sources, "refs/heads/" + pages); + } + + public String[] asRefArray() { + return asRefList().toArray(String[]::new); } } } diff --git a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java index 2454e9ed..5ce6b750 100644 --- a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java +++ b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java @@ -99,8 +99,8 @@ public QuarkusIO(QuarkusIOConfig config, GitCloneDirectory mainRepository, @Override public void close() throws Exception { try (var closer = new Closer()) { - closer.push(GitCloneDirectory::close, mainRepository); closer.push(CloseableDirectory::close, prefetchedQuarkiverseGuides); + closer.push(GitCloneDirectory::close, mainRepository); closer.pushAll(GitCloneDirectory::close, localizedSites.values()); } } @@ -113,7 +113,7 @@ public Stream guides() throws IOException { // guides based on the info from the _data/versioned/[version]/index/ // may contain quarkus.yaml as well as quarkiverse.yml private Stream versionedGuides() throws IOException { - return Files.list(mainRepository.directory().path().resolve("_data").resolve("versioned")) + return Files.list(mainRepository.resolve("_data").resolve("versioned")) .flatMap(p -> { var version = p.getFileName().toString().replace('-', '.'); Path quarkiverse = p.resolve("index").resolve("quarkiverse.yaml"); @@ -143,13 +143,13 @@ private Stream versionedGuides() throws IOException { private static Path resolveTranslationPath(String version, String filename, GitCloneDirectory directory, Language language) { - return directory.directory().path().resolve( + return directory.resolve( Path.of("l10n", "po", language.locale, "_data", "versioned", version, "index", filename + ".po")); } // older version guides like guides-2-7.yaml or guides-2-13.yaml private Stream legacyGuides() throws IOException { - return Files.list(mainRepository.directory().path().resolve("_data")) + return Files.list(mainRepository.resolve("_data")) .filter(p -> !Files.isDirectory(p) && p.getFileName().toString().startsWith("guides-")) .flatMap(p -> { var version = p.getFileName().toString().replaceAll("guides-|\\.yaml", "").replace('-', '.'); @@ -164,7 +164,7 @@ private Stream legacyGuides() throws IOException { } private static Path resolveLegacyTranslationPath(String filename, GitCloneDirectory directory, Language language) { - return directory.directory().path().resolve( + return directory.resolve( Path.of("l10n", "po", language.locale, "_data", filename + ".po")); } diff --git a/src/main/java/io/quarkus/search/app/util/CloseableDirectory.java b/src/main/java/io/quarkus/search/app/util/CloseableDirectory.java index ee9f023f..ab100142 100644 --- a/src/main/java/io/quarkus/search/app/util/CloseableDirectory.java +++ b/src/main/java/io/quarkus/search/app/util/CloseableDirectory.java @@ -36,4 +36,11 @@ public Path path() { return path; } + @Override + public String toString() { + return "CloseableDirectory{" + + "path=" + path + + ", shouldDelete=" + shouldDelete + + '}'; + } } diff --git a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java index 62b76ad3..b2c8fde6 100644 --- a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java +++ b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java @@ -2,39 +2,71 @@ import java.io.Closeable; import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; +import java.util.List; + +import io.quarkus.search.app.fetching.FetchingService; +import io.quarkus.search.app.fetching.LoggerProgressMonitor; + +import io.quarkus.logging.Log; import org.hibernate.search.util.common.impl.Closer; +import org.hibernate.search.util.common.impl.SuppressingCloser; import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.transport.RemoteConfig; +import org.jboss.logging.Logger; public class GitCloneDirectory implements Closeable { + + private static final Logger log = Logger.getLogger(GitCloneDirectory.class); private final Git git; - private final CloseableDirectory directory; - private final String pagesBranch; + private final GitDirectoryDetails directory; private RevTree pagesTree; - public GitCloneDirectory(Git git, CloseableDirectory directory, String pagesBranch) { + public GitCloneDirectory(Git git, GitDirectoryDetails directory) { this.git = git; this.directory = directory; - this.pagesBranch = pagesBranch; + } + + public GitCloneDirectory(Git git, Path path, String pages) { + this(git, new GitDirectoryDetails(path, pages)); } public Git git() { return git; } - public CloseableDirectory directory() { + public Path resolve(String other) { + return directory.directory().resolve(other); + } + + public GitDirectoryDetails directory() { return directory; } + public Path resolve(Path other) { + return directory.directory().resolve(other); + } + public RevTree pagesTree() { if (this.pagesTree == null) { try { - this.pagesTree = GitUtils.firstExistingRevTree(git().getRepository(), "origin/" + pagesBranch); - } catch (IOException e) { - throw new RuntimeException("Unable to locate pages branch: " + pagesBranch, e); + String remote; + List remotes = git.remoteList().call(); + if (remotes.isEmpty()) { + remote = ""; + } else { + remote = remotes.get(0).getName() + "/"; + } + + this.pagesTree = GitUtils.firstExistingRevTree(git.getRepository(), remote + directory.pagesBranch()); + } catch (GitAPIException | IOException e) { + throw new RuntimeException("Unable to locate pages branch: " + directory.pagesBranch(), e); } } return pagesTree; @@ -44,8 +76,74 @@ public RevTree pagesTree() { public void close() throws IOException { try (Closer closer = new Closer<>()) { closer.push(Git::close, git); - closer.push(CloseableDirectory::close, directory); + pagesTree = null; } } + @Override + public String toString() { + return "GitCloneDirectory{" + + "git=" + git + + ", directory=" + directory + + '}'; + } + + public record GitDirectoryDetails(Path directory, String pagesBranch) { + public GitCloneDirectory open() throws IOException { + return new GitCloneDirectory(Git.open(directory.toFile()), this); + } + + public GitCloneDirectory pull(FetchingService.Branches branches) { + Log.infof("Pulling changes for '%s'.", directory); + Git git = null; + try { + git = Git.open(directory.toFile()); + Log.infof("Pulling changes for sources branch of '%s':'%s'.", directory, branches.sources()); + // just to make sure we are in the correct branch: + git.checkout().setName(branches.sources()).call(); + // pull sources branch + git.pull() + .setProgressMonitor(LoggerProgressMonitor.create(log, + "Pulling into '" + directory + "' (" + branches.sources() + "): ")) + .call(); + // fetch used branches + git.fetch().setForceUpdate(true) + .setProgressMonitor(LoggerProgressMonitor.create(log, + "Fetching into '" + directory + "' (" + branches.sources() + "): ")) + .setRefSpecs(branches.asRefArray()) + .call(); + return new GitCloneDirectory(git, this); + } catch (RuntimeException | GitAPIException | IOException e) { + new SuppressingCloser(e).push(git); + throw new IllegalStateException( + "Wasn't able to pull changes for branch '%s' in repository '%s': '%s".formatted(branches, + directory, + e.getMessage()), + e); + } + } + + public GitCloneDirectory clone(URI gitUri, FetchingService.Branches branches) { + Log.infof("Cloning into '%s' from '%s'.", directory, gitUri); + Git git = null; + try { + git = Git.cloneRepository() + .setURI(gitUri.toString()) + .setDirectory(directory.toFile()) + .setDepth(1) + .setNoTags() + .setBranch(branches.sources()) + .setBranchesToClone(branches.asRefList()) + .setProgressMonitor(LoggerProgressMonitor.create(log, "Cloning " + gitUri + ": ")) + // Unfortunately sparse checkouts are not supported: https://www.eclipse.org/forums/index.php/t/1094825/ + .call(); + return new GitCloneDirectory(git, this); + } catch (RuntimeException | GitAPIException e) { + new SuppressingCloser(e).push(git); + throw new IllegalStateException( + "Failed to clone git repository into '%s' from '%s': %s".formatted(directory, gitUri, e.getMessage()), + e); + } + } + } } diff --git a/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java b/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java index 31aa9887..f14530b6 100644 --- a/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java +++ b/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java @@ -139,6 +139,27 @@ static void initOrigin() throws IOException, GitAPIException { } } + private void updateMainRepository() throws IOException, GitAPIException { + Path sourceRepoPath = tmpDir.path(); + Path metadata1ToFetch = sourceRepoPath.resolve("_data/versioned/latest/index/quarkus.yaml"); + Path guide1HtmlToFetch = sourceRepoPath.resolve("guides/" + FETCHED_GUIDE_1_NAME + ".html"); + + try (Git git = Git.open(sourceRepoPath.toFile())) { + GitTestUtils.cleanGitUserConfig(); + + git.checkout().setName(QuarkusIO.PAGES_BRANCH).call(); + Files.writeString(guide1HtmlToFetch, FETCHED_GUIDE_1_CONTENT_HTML_UPDATED); + git.add().addFilepattern(".").call(); + git.commit().setMessage("Pages updated commit").call(); + + git.checkout().setName(QuarkusIO.SOURCE_BRANCH).call(); + + Files.writeString(metadata1ToFetch, METADATA_YAML_UPDATED); + git.add().addFilepattern(".").call(); + git.commit().setMessage("Source updated commit").call(); + } + } + @AfterAll static void deleteTmpDir() throws IOException { try (Closer closer = new Closer<>()) { @@ -150,6 +171,10 @@ static void deleteTmpDir() throws IOException { @RegisterExtension static final QuarkusComponentTestExtension extension = QuarkusComponentTestExtension.builder() .configProperty("fetching.timeout", "PT30s") + // NOTE: a more correct way to define these URIs would've been `dir.path().toUri().toString()` + // so that schema and everything is added to a string and it is a correct URL. + // We do not do it like that to trick the app into thinking that it is supplied with a non-local repository + // and that it needs to clone it. .configProperty("quarkusio.git-uri", tmpDir.path().toString()) .configProperty("quarkusio.localized.es.git-uri", localizedDirectories.get(Language.SPANISH).path().toString()) .configProperty("quarkusio.localized.pt.git-uri", localizedDirectories.get(Language.PORTUGUESE).path().toString()) @@ -252,6 +277,104 @@ void fetchQuarkusIo() throws Exception { JA_FETCHED_GUIDE_2_CONTENT_HTML)); } } + // now let's update some guides and make sure that the content is fetched correctly: + updateMainRepository(); + + // NOTE that after an update we'll have non-translated titles and summaries, + // since in this test we've only updated them in the "main" repository, + // and as a result there's no translation for them in localized sites. + // Content file though is still the one from the localized site! + // + try (QuarkusIO quarkusIO = service.fetchQuarkusIo()) { + try (var guides = quarkusIO.guides()) { + assertThat(guides) + .hasSize(10) + .satisfiesExactlyInAnyOrder( + isGuide("https://quarkus.io/guides/" + FETCHED_GUIDE_1_NAME, + "Some updated title", + "This is an updated summary", + "keyword1 keyword2", + Set.of("category1", "category2"), + Set.of("topic1", "topic2"), + Set.of("io.quarkus:extension1", "io.quarkus:extension2"), + FETCHED_GUIDE_1_CONTENT_HTML_UPDATED), + isGuide("https://cn.quarkus.io/guides/" + FETCHED_GUIDE_1_NAME, + "Some updated title", + "This is an updated summary", + "keyword1 keyword2", + Set.of("category1", "category2"), + Set.of("topic1", "topic2"), + Set.of("io.quarkus:extension1", "io.quarkus:extension2"), + JA_FETCHED_GUIDE_1_CONTENT_HTML), + isGuide("https://es.quarkus.io/guides/" + FETCHED_GUIDE_1_NAME, + "Some updated title", + "This is an updated summary", + "keyword1 keyword2", + Set.of("category1", "category2"), + Set.of("topic1", "topic2"), + Set.of("io.quarkus:extension1", "io.quarkus:extension2"), + JA_FETCHED_GUIDE_1_CONTENT_HTML), + isGuide("https://ja.quarkus.io/guides/" + FETCHED_GUIDE_1_NAME, + "Some updated title", + "This is an updated summary", + "keyword1 keyword2", + Set.of("category1", "category2"), + Set.of("topic1", "topic2"), + Set.of("io.quarkus:extension1", "io.quarkus:extension2"), + JA_FETCHED_GUIDE_1_CONTENT_HTML), + isGuide("https://pt.quarkus.io/guides/" + FETCHED_GUIDE_1_NAME, + "Some updated title", + "This is an updated summary", + "keyword1 keyword2", + Set.of("category1", "category2"), + Set.of("topic1", "topic2"), + Set.of("io.quarkus:extension1", "io.quarkus:extension2"), + JA_FETCHED_GUIDE_1_CONTENT_HTML), + + isGuide("https://quarkus.io/version/2.7/guides/" + FETCHED_GUIDE_2_NAME, + "Some other title", + "This is a different summary.", + null, + Set.of("getting-started"), + Set.of(), + Set.of(), + FETCHED_GUIDE_2_CONTENT_HTML), + isGuide("https://cn.quarkus.io/version/2.7/guides/" + FETCHED_GUIDE_2_NAME, + "Some other title", + // Even though there's a translation available it is "fuzzy", hence we ignore it + // and use the original message: + "This is a different summary.", + null, + Set.of("getting-started"), + Set.of(), + Set.of(), + JA_FETCHED_GUIDE_2_CONTENT_HTML), + isGuide("https://es.quarkus.io/version/2.7/guides/" + FETCHED_GUIDE_2_NAME, + "Some other title", + "This is a different summary.", + null, + Set.of("getting-started"), + Set.of(), + Set.of(), + JA_FETCHED_GUIDE_2_CONTENT_HTML), + isGuide("https://ja.quarkus.io/version/2.7/guides/" + FETCHED_GUIDE_2_NAME, + "Some other title", + "This is a different summary.", + null, + Set.of("getting-started"), + Set.of(), + Set.of(), + JA_FETCHED_GUIDE_2_CONTENT_HTML), + isGuide("https://pt.quarkus.io/version/2.7/guides/" + FETCHED_GUIDE_2_NAME, + "Some other title", + "This is a different summary.", + null, + Set.of("getting-started"), + Set.of(), + Set.of(), + JA_FETCHED_GUIDE_2_CONTENT_HTML)); + } + } } private static final String METADATA_YAML = """ @@ -275,6 +398,27 @@ void fetchQuarkusIo() throws Exception { url: /guides/foo """; + private static final String METADATA_YAML_UPDATED = """ + # Generated file. Do not edit + --- + types: + reference: + - title: Some updated title + filename: foo.adoc + summary: This is an updated summary + categories: "category1, category2" + keywords: keyword1 keyword2 + topics: + - topic1 + - topic2 + extensions: + - io.quarkus:extension1 + - io.quarkus:extension2 + id: foo + type: reference + url: /guides/foo + """; + private static final String METADATA_LEGACY_YAML = """ # Generated file. Do not edit --- @@ -299,6 +443,18 @@ void fetchQuarkusIo() throws Exception {

Some other subsection

This is another subsection """; + + private static final String FETCHED_GUIDE_1_CONTENT_HTML_UPDATED = """ + + + +

Some title

+

This is the updated guide body +

Some updated subsection

+ This is an updated subsection +

Some other updated subsection

+ This is another updated subsection + """; private static final String FETCHED_GUIDE_2_NAME = "bar"; private static final String FETCHED_GUIDE_2_CONTENT_HTML = """ From ef58bd5458494224a05c43b805334b840b5aa284 Mon Sep 17 00:00:00 2001 From: marko-bekhta Date: Wed, 10 Jan 2024 11:58:08 +0100 Subject: [PATCH 2/8] Make sure a correct refs are inspected for pages tree --- .../search/app/fetching/FetchingService.java | 4 +- .../search/app/quarkusio/QuarkusIO.java | 2 +- .../search/app/util/GitCloneDirectory.java | 43 ++++++++++++++++--- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java index 7959a41b..b6e7db8d 100644 --- a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java +++ b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java @@ -117,8 +117,8 @@ private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branch repository = new GitCloneDirectory.GitDirectoryDetails(cloneDir.path(), branches.pages()); repositories.put(requestedGitUri, repository); - // If we have a local repository -- just open it, clone it otherwise: - return requiresCloning ? repository.clone(gitUri, branches) : repository.open(); + // If we have a local repository -- open it, and then pull the changes, clone it otherwise: + return requiresCloning ? repository.clone(gitUri, branches) : repository.open().update(branches); } catch (RuntimeException | IOException e) { new SuppressingCloser(e).push(cloneDir); throw new IllegalStateException("Failed to fetch '%s': %s".formatted(siteName, e.getMessage()), e); diff --git a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java index 5ce6b750..277688d3 100644 --- a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java +++ b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java @@ -275,7 +275,7 @@ private Stream translateGuide(Guide guide, Map ORDERED_REMOTES = Arrays.asList("origin", "upstream"); private final Git git; private final GitDirectoryDetails directory; @@ -56,22 +58,39 @@ public Path resolve(Path other) { public RevTree pagesTree() { if (this.pagesTree == null) { try { - String remote; - List remotes = git.remoteList().call(); + String remote = null; + Set remotes = git.getRepository().getRemoteNames(); if (remotes.isEmpty()) { remote = ""; } else { - remote = remotes.get(0).getName() + "/"; + for (String name : ORDERED_REMOTES) { + if (remotes.contains(name)) { + remote = name + "/"; + } + } + if (remote == null) { + log.warn( + "Wasn't able to find any of the default/expected remotes (%s) so a random existing one will be picked. Indexing results are not guaranteed to be correct." + .formatted(ORDERED_REMOTES)); + // If at this point we still haven't figured out the remote ... then we probably are going to fail anyway, + // but let's give it one more chance and just pick any of the remotes at random. + remote = remotes.iterator().next() + "/"; + } } this.pagesTree = GitUtils.firstExistingRevTree(git.getRepository(), remote + directory.pagesBranch()); - } catch (GitAPIException | IOException e) { + } catch (IOException e) { throw new RuntimeException("Unable to locate pages branch: " + directory.pagesBranch(), e); } } return pagesTree; } + public GitCloneDirectory update(FetchingService.Branches branches) { + directory.pull(git, branches); + return this; + } + @Override public void close() throws IOException { try (Closer closer = new Closer<>()) { @@ -98,6 +117,17 @@ public GitCloneDirectory pull(FetchingService.Branches branches) { Git git = null; try { git = Git.open(directory.toFile()); + pull(git, branches); + return new GitCloneDirectory(git, this); + } catch (IOException e) { + new SuppressingCloser(e).push(git); + throw new IllegalStateException("Wasn't able to open repository '%s': '%s".formatted(directory, e.getMessage()), + e); + } + } + + private void pull(Git git, FetchingService.Branches branches) { + try { Log.infof("Pulling changes for sources branch of '%s':'%s'.", directory, branches.sources()); // just to make sure we are in the correct branch: git.checkout().setName(branches.sources()).call(); @@ -112,8 +142,7 @@ public GitCloneDirectory pull(FetchingService.Branches branches) { "Fetching into '" + directory + "' (" + branches.sources() + "): ")) .setRefSpecs(branches.asRefArray()) .call(); - return new GitCloneDirectory(git, this); - } catch (RuntimeException | GitAPIException | IOException e) { + } catch (RuntimeException | GitAPIException e) { new SuppressingCloser(e).push(git); throw new IllegalStateException( "Wasn't able to pull changes for branch '%s' in repository '%s': '%s".formatted(branches, From a467a17479b9e22e767fc426727614bdc1fc3362 Mon Sep 17 00:00:00 2001 From: marko-bekhta Date: Wed, 10 Jan 2024 11:58:40 +0100 Subject: [PATCH 3/8] Make sure that local repo has an expected branch checked out --- .../search/app/fetching/FetchingService.java | 4 ++-- .../search/app/util/GitCloneDirectory.java | 20 +++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java index b6e7db8d..54c74076 100644 --- a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java +++ b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java @@ -89,7 +89,7 @@ private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branch if (repository != null) { // We are working with a zip file, so we have nothing to refresh as there's no actual remote available; // just return the same repository without any changes: - return repository.open(); + return repository.open(branches.sources()); } Log.warnf("Unzipping '%s': this application is most likely indexing only a sample of %s." @@ -118,7 +118,7 @@ private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branch repositories.put(requestedGitUri, repository); // If we have a local repository -- open it, and then pull the changes, clone it otherwise: - return requiresCloning ? repository.clone(gitUri, branches) : repository.open().update(branches); + return requiresCloning ? repository.clone(gitUri, branches) : repository.open(branches.sources()).update(branches); } catch (RuntimeException | IOException e) { new SuppressingCloser(e).push(cloneDir); throw new IllegalStateException("Failed to fetch '%s': %s".formatted(siteName, e.getMessage()), e); diff --git a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java index fea3db0d..55a016d0 100644 --- a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java +++ b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java @@ -108,8 +108,24 @@ public String toString() { } public record GitDirectoryDetails(Path directory, String pagesBranch) { - public GitCloneDirectory open() throws IOException { - return new GitCloneDirectory(Git.open(directory.toFile()), this); + public GitCloneDirectory open(String sources) throws IOException { + Git git = null; + try { + git = Git.open(directory.toFile()); + // and let's make sure we are in a correct branch: + if (!sources.equals(git.getRepository().getBranch())) { + git.checkout() + .setName(sources) + .setProgressMonitor(LoggerProgressMonitor.create(log, + "Checking out ('%s') in repository '%s'".formatted(sources, directory))) + .call(); + } + return new GitCloneDirectory(git, this); + } catch (GitAPIException e) { + new SuppressingCloser(e).push(git); + throw new IllegalStateException( + "Wasn't able to open '%s' as a git repository: '%s".formatted(directory, e.getMessage()), e); + } } public GitCloneDirectory pull(FetchingService.Branches branches) { From 8f227cd8b1f738b0ec6fdd1462031334cc7b4f04 Mon Sep 17 00:00:00 2001 From: marko-bekhta Date: Wed, 10 Jan 2024 15:07:43 +0100 Subject: [PATCH 4/8] Do not update the repositories if there's no remote --- .../java/io/quarkus/search/app/util/GitCloneDirectory.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java index 55a016d0..7bc7b91e 100644 --- a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java +++ b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java @@ -147,6 +147,13 @@ private void pull(Git git, FetchingService.Branches branches) { Log.infof("Pulling changes for sources branch of '%s':'%s'.", directory, branches.sources()); // just to make sure we are in the correct branch: git.checkout().setName(branches.sources()).call(); + + if (git.getRepository().getRemoteNames().isEmpty()) { + // Well... then there's nowhere to pull from, + // and we exit faster as pulling or fetching will fail in this scenario. + return; + } + // pull sources branch git.pull() .setProgressMonitor(LoggerProgressMonitor.create(log, From 2bba01fb33483b89623948cfe04ecfea058522a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Wed, 10 Jan 2024 16:44:32 +0100 Subject: [PATCH 5/8] Fetch the right remote when updating local clones --- .../search/app/fetching/FetchingService.java | 17 +-- .../search/app/util/GitCloneDirectory.java | 115 +++++++----------- 2 files changed, 51 insertions(+), 81 deletions(-) diff --git a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java index 54c74076..ad054be1 100644 --- a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java +++ b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java @@ -84,14 +84,11 @@ private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branch CloseableDirectory cloneDir = null; try { GitCloneDirectory.GitDirectoryDetails repository = repositories.get(gitUri); + if (repository != null) { + return repository.pull(branches); + } if (LaunchMode.DEVELOPMENT.equals(LaunchMode.current()) && isZip(gitUri)) { - if (repository != null) { - // We are working with a zip file, so we have nothing to refresh as there's no actual remote available; - // just return the same repository without any changes: - return repository.open(branches.sources()); - } - Log.warnf("Unzipping '%s': this application is most likely indexing only a sample of %s." + " See README to index the full website.", gitUri, siteName); @@ -100,12 +97,6 @@ private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branch gitUri = unzippedPath.toUri(); // Fall-through and clone the directory. // This cloning is as if we are cloning a remote repository. - } else { - if (repository != null) { - // It's not a zip, so it may be either a local directory or a remote repository - // let's pull the changes from it as it shouldn't cause any exceptions, as a remote is actually out there. - return repository.pull(branches); - } } cloneDir = requiresCloning @@ -118,7 +109,7 @@ private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branch repositories.put(requestedGitUri, repository); // If we have a local repository -- open it, and then pull the changes, clone it otherwise: - return requiresCloning ? repository.clone(gitUri, branches) : repository.open(branches.sources()).update(branches); + return requiresCloning ? repository.clone(gitUri, branches) : repository.pull(branches); } catch (RuntimeException | IOException e) { new SuppressingCloser(e).push(cloneDir); throw new IllegalStateException("Failed to fetch '%s': %s".formatted(siteName, e.getMessage()), e); diff --git a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java index 7bc7b91e..25ddcb6d 100644 --- a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java +++ b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java @@ -24,19 +24,17 @@ public class GitCloneDirectory implements Closeable { private static final Logger log = Logger.getLogger(GitCloneDirectory.class); - private static final List ORDERED_REMOTES = Arrays.asList("origin", "upstream"); + private static final List ORDERED_REMOTES = Arrays.asList("upstream", "origin"); private final Git git; private final GitDirectoryDetails directory; + private final String remoteName; private RevTree pagesTree; - public GitCloneDirectory(Git git, GitDirectoryDetails directory) { + public GitCloneDirectory(Git git, GitDirectoryDetails directory, String remoteName) { this.git = git; this.directory = directory; - } - - public GitCloneDirectory(Git git, Path path, String pages) { - this(git, new GitDirectoryDetails(path, pages)); + this.remoteName = remoteName; } public Git git() { @@ -58,27 +56,8 @@ public Path resolve(Path other) { public RevTree pagesTree() { if (this.pagesTree == null) { try { - String remote = null; - Set remotes = git.getRepository().getRemoteNames(); - if (remotes.isEmpty()) { - remote = ""; - } else { - for (String name : ORDERED_REMOTES) { - if (remotes.contains(name)) { - remote = name + "/"; - } - } - if (remote == null) { - log.warn( - "Wasn't able to find any of the default/expected remotes (%s) so a random existing one will be picked. Indexing results are not guaranteed to be correct." - .formatted(ORDERED_REMOTES)); - // If at this point we still haven't figured out the remote ... then we probably are going to fail anyway, - // but let's give it one more chance and just pick any of the remotes at random. - remote = remotes.iterator().next() + "/"; - } - } - - this.pagesTree = GitUtils.firstExistingRevTree(git.getRepository(), remote + directory.pagesBranch()); + this.pagesTree = GitUtils.firstExistingRevTree(git.getRepository(), + (remoteName == null ? "" : remoteName + "/") + directory.pagesBranch()); } catch (IOException e) { throw new RuntimeException("Unable to locate pages branch: " + directory.pagesBranch(), e); } @@ -86,11 +65,6 @@ public RevTree pagesTree() { return pagesTree; } - public GitCloneDirectory update(FetchingService.Branches branches) { - directory.pull(git, branches); - return this; - } - @Override public void close() throws IOException { try (Closer closer = new Closer<>()) { @@ -108,11 +82,13 @@ public String toString() { } public record GitDirectoryDetails(Path directory, String pagesBranch) { - public GitCloneDirectory open(String sources) throws IOException { + public GitCloneDirectory pull(FetchingService.Branches branches) { + Log.infof("Pulling changes for '%s'.", directory); Git git = null; try { git = Git.open(directory.toFile()); - // and let's make sure we are in a correct branch: + // let's make sure we are in a correct branch: + String sources = branches.sources(); if (!sources.equals(git.getRepository().getBranch())) { git.checkout() .setName(sources) @@ -120,51 +96,34 @@ public GitCloneDirectory open(String sources) throws IOException { "Checking out ('%s') in repository '%s'".formatted(sources, directory))) .call(); } - return new GitCloneDirectory(git, this); - } catch (GitAPIException e) { - new SuppressingCloser(e).push(git); - throw new IllegalStateException( - "Wasn't able to open '%s' as a git repository: '%s".formatted(directory, e.getMessage()), e); - } - } - - public GitCloneDirectory pull(FetchingService.Branches branches) { - Log.infof("Pulling changes for '%s'.", directory); - Git git = null; - try { - git = Git.open(directory.toFile()); - pull(git, branches); - return new GitCloneDirectory(git, this); - } catch (IOException e) { + String remoteName = inferRemoteName(git); + if (remoteName != null) { + pull(git, branches, remoteName); + } + // Else there's nowhere to pull from, so we don't even try. + return new GitCloneDirectory(git, this, remoteName); + } catch (IOException | GitAPIException e) { new SuppressingCloser(e).push(git); throw new IllegalStateException("Wasn't able to open repository '%s': '%s".formatted(directory, e.getMessage()), e); } } - private void pull(Git git, FetchingService.Branches branches) { + private void pull(Git git, FetchingService.Branches branches, String remoteName) { try { - Log.infof("Pulling changes for sources branch of '%s':'%s'.", directory, branches.sources()); - // just to make sure we are in the correct branch: - git.checkout().setName(branches.sources()).call(); - - if (git.getRepository().getRemoteNames().isEmpty()) { - // Well... then there's nowhere to pull from, - // and we exit faster as pulling or fetching will fail in this scenario. - return; - } + // fetch remote branches to make sure we'll use up-to-date data + git.fetch() + .setRemote(remoteName) + .setRefSpecs(branches.asRefArray()) + .setProgressMonitor(LoggerProgressMonitor.create(log, + "Fetching into '" + directory + "' (" + branches.pages() + "): ")) + .call(); - // pull sources branch + // pull the sources branch, to update the working directory git.pull() .setProgressMonitor(LoggerProgressMonitor.create(log, "Pulling into '" + directory + "' (" + branches.sources() + "): ")) .call(); - // fetch used branches - git.fetch().setForceUpdate(true) - .setProgressMonitor(LoggerProgressMonitor.create(log, - "Fetching into '" + directory + "' (" + branches.sources() + "): ")) - .setRefSpecs(branches.asRefArray()) - .call(); } catch (RuntimeException | GitAPIException e) { new SuppressingCloser(e).push(git); throw new IllegalStateException( @@ -175,13 +134,33 @@ private void pull(Git git, FetchingService.Branches branches) { } } + private String inferRemoteName(Git git) { + Set remotes = git.getRepository().getRemoteNames(); + if (remotes.isEmpty()) { + return null; + } + for (String name : ORDERED_REMOTES) { + if (remotes.contains(name)) { + return name; + } + } + log.warn( + "Wasn't able to find any of the default/expected remotes (%s) so a random existing one will be picked. Indexing results are not guaranteed to be correct." + .formatted(ORDERED_REMOTES)); + // If at this point we still haven't figured out the remote ... then we probably are going to fail anyway, + // but let's give it one more chance and just pick any of the remotes at random. + return remotes.iterator().next(); + } + public GitCloneDirectory clone(URI gitUri, FetchingService.Branches branches) { Log.infof("Cloning into '%s' from '%s'.", directory, gitUri); Git git = null; + String remoteName = ORDERED_REMOTES.get(0); try { git = Git.cloneRepository() .setURI(gitUri.toString()) .setDirectory(directory.toFile()) + .setRemote(remoteName) .setDepth(1) .setNoTags() .setBranch(branches.sources()) @@ -189,7 +168,7 @@ public GitCloneDirectory clone(URI gitUri, FetchingService.Branches branches) { .setProgressMonitor(LoggerProgressMonitor.create(log, "Cloning " + gitUri + ": ")) // Unfortunately sparse checkouts are not supported: https://www.eclipse.org/forums/index.php/t/1094825/ .call(); - return new GitCloneDirectory(git, this); + return new GitCloneDirectory(git, this, remoteName); } catch (RuntimeException | GitAPIException e) { new SuppressingCloser(e).push(git); throw new IllegalStateException( From b7f5c42e1468ad96ff754e7e9416a06e10fc61db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Wed, 10 Jan 2024 17:08:16 +0100 Subject: [PATCH 6/8] Always perform git cloning of local repos in prod/tests, and never in dev Prod will always use remote (http/git) git URIs, never local ones (file). So prod will always clone for sure, and we want tests to do the same. --- .../search/app/fetching/FetchingService.java | 61 ++++++++++--------- .../app/fetching/FetchingServiceTest.java | 18 +++--- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java index ad054be1..411e3079 100644 --- a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java +++ b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java @@ -49,17 +49,14 @@ public class FetchingService { public QuarkusIO fetchQuarkusIo() { CompletableFuture main = null; Map> localized = new LinkedHashMap<>(); - try (CloseableDirectory unzipDir = LaunchMode.DEVELOPMENT.equals(LaunchMode.current()) - ? CloseableDirectory.temp("quarkus.io-unzipped") - : null; - SimpleExecutor executor = new SimpleExecutor(fetchingConfig.parallelism())) { - main = executor.submit(() -> fetchQuarkusIoSite("quarkus.io", quarkusIOConfig.gitUri(), MAIN, unzipDir)); + try (SimpleExecutor executor = new SimpleExecutor(fetchingConfig.parallelism())) { + main = executor.submit(() -> fetchQuarkusIoSite("quarkus.io", quarkusIOConfig.gitUri(), MAIN)); for (Map.Entry entry : sortMap(quarkusIOConfig.localized()).entrySet()) { var language = entry.getKey(); var config = entry.getValue(); localized.put(language, executor.submit( - () -> fetchQuarkusIoSite(language.code + ".quarkus.io", config.gitUri(), LOCALIZED, unzipDir))); + () -> fetchQuarkusIoSite(language.code + ".quarkus.io", config.gitUri(), LOCALIZED))); } executor.waitForSuccessOrThrow(fetchingConfig.timeout()); // If we get here, all tasks succeeded. @@ -73,45 +70,51 @@ public QuarkusIO fetchQuarkusIo() { } } - private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branches branches, - CloseableDirectory unzipDir) { - // We only want to clean up a clone directory if it is a clone from a remote repository or an unzipped one. - // If we got a local repository we want to keep it as is and no cloning is needed. - // If it was a local unzipped version -- the "cloneDir" will be removed sooner than we are done with indexing, - // so we want to "clone" that directory, and then it'll be removed as if a remote repository clone. - boolean requiresCloning = !isFile(gitUri) || isZip(gitUri); + private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branches branches) { URI requestedGitUri = gitUri; - CloseableDirectory cloneDir = null; + CloseableDirectory repoDir = null; try { GitCloneDirectory.GitDirectoryDetails repository = repositories.get(gitUri); if (repository != null) { return repository.pull(branches); } - if (LaunchMode.DEVELOPMENT.equals(LaunchMode.current()) && isZip(gitUri)) { - Log.warnf("Unzipping '%s': this application is most likely indexing only a sample of %s." - + " See README to index the full website.", - gitUri, siteName); - Path unzippedPath = unzipDir.path().resolve(siteName); - unzip(Path.of(gitUri), unzippedPath); - gitUri = unzippedPath.toUri(); - // Fall-through and clone the directory. - // This cloning is as if we are cloning a remote repository. + if (LaunchMode.DEVELOPMENT.equals(LaunchMode.current())) { + if (isZip(gitUri)) { + Log.warnf("Unzipping '%s': this application is most likely indexing only a sample of %s." + + " See README to index the full website.", + gitUri, siteName); + repoDir = CloseableDirectory.temp(siteName); + unzip(Path.of(gitUri), repoDir.path()); + } else if (isFile(gitUri)) { + Log.infof("Using the git repository '%s' as-is without cloning to speed up indexing of %s.", + gitUri, siteName); + // In dev mode, we want to skip cloning when possible, to make things quicker. + repoDir = CloseableDirectory.of(Paths.get(gitUri)); + } } - cloneDir = requiresCloning - ? CloseableDirectory.temp(siteName) - : CloseableDirectory.of(Paths.get(gitUri)); + boolean requiresCloning; + if (repoDir == null) { + // We always end up here in prod and tests. + // That's fine, because prod will always use remote (http/git) git URIs anyway, + // never local ones (file). + repoDir = CloseableDirectory.temp(siteName); + requiresCloning = true; + } else { + // We may end up here, but only in dev mode + requiresCloning = false; + } - closeableDirectories.add(cloneDir); + closeableDirectories.add(repoDir); - repository = new GitCloneDirectory.GitDirectoryDetails(cloneDir.path(), branches.pages()); + repository = new GitCloneDirectory.GitDirectoryDetails(repoDir.path(), branches.pages()); repositories.put(requestedGitUri, repository); // If we have a local repository -- open it, and then pull the changes, clone it otherwise: return requiresCloning ? repository.clone(gitUri, branches) : repository.pull(branches); } catch (RuntimeException | IOException e) { - new SuppressingCloser(e).push(cloneDir); + new SuppressingCloser(e).push(repoDir); throw new IllegalStateException("Failed to fetch '%s': %s".formatted(siteName, e.getMessage()), e); } } diff --git a/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java b/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java index f14530b6..425eb6d1 100644 --- a/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java +++ b/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java @@ -171,15 +171,15 @@ static void deleteTmpDir() throws IOException { @RegisterExtension static final QuarkusComponentTestExtension extension = QuarkusComponentTestExtension.builder() .configProperty("fetching.timeout", "PT30s") - // NOTE: a more correct way to define these URIs would've been `dir.path().toUri().toString()` - // so that schema and everything is added to a string and it is a correct URL. - // We do not do it like that to trick the app into thinking that it is supplied with a non-local repository - // and that it needs to clone it. - .configProperty("quarkusio.git-uri", tmpDir.path().toString()) - .configProperty("quarkusio.localized.es.git-uri", localizedDirectories.get(Language.SPANISH).path().toString()) - .configProperty("quarkusio.localized.pt.git-uri", localizedDirectories.get(Language.PORTUGUESE).path().toString()) - .configProperty("quarkusio.localized.cn.git-uri", localizedDirectories.get(Language.CHINESE).path().toString()) - .configProperty("quarkusio.localized.ja.git-uri", localizedDirectories.get(Language.JAPANESE).path().toString()) + .configProperty("quarkusio.git-uri", tmpDir.path().toUri().toString()) + .configProperty("quarkusio.localized.es.git-uri", + localizedDirectories.get(Language.SPANISH).path().toUri().toString()) + .configProperty("quarkusio.localized.pt.git-uri", + localizedDirectories.get(Language.PORTUGUESE).path().toUri().toString()) + .configProperty("quarkusio.localized.cn.git-uri", + localizedDirectories.get(Language.CHINESE).path().toUri().toString()) + .configProperty("quarkusio.localized.ja.git-uri", + localizedDirectories.get(Language.JAPANESE).path().toUri().toString()) .build(); @Inject From 5ef67dfe7458c2b073275abc81c4562f8e55ede4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Wed, 10 Jan 2024 18:21:41 +0100 Subject: [PATCH 7/8] Refactor git cloning/updating code The goal was to make this code a bit more straightforward... I'm not sure that's a complete success :) --- .../search/app/fetching/FetchingService.java | 69 +++++-------- .../search/app/quarkusio/QuarkusIO.java | 8 +- .../search/app/util/GitCloneDirectory.java | 97 +++++++++++-------- .../app/fetching/FetchingServiceTest.java | 12 +-- .../app/testsupport/QuarkusIOSample.java | 28 +++--- 5 files changed, 107 insertions(+), 107 deletions(-) diff --git a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java index 411e3079..cc36e026 100644 --- a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java +++ b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java @@ -5,11 +5,9 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -34,8 +32,6 @@ @ApplicationScoped public class FetchingService { - private static final Branches MAIN = new Branches(QuarkusIO.SOURCE_BRANCH, QuarkusIO.PAGES_BRANCH); - private static final Branches LOCALIZED = new Branches(QuarkusIO.LOCALIZED_SOURCE_BRANCH, QuarkusIO.LOCALIZED_PAGES_BRANCH); @Inject FetchingConfig fetchingConfig; @@ -43,20 +39,20 @@ public class FetchingService { @Inject QuarkusIOConfig quarkusIOConfig; - private final Map repositories = new HashMap<>(); - private final Set closeableDirectories = new HashSet<>(); + private final Map detailsCache = new HashMap<>(); + private final Set tempDirectories = new HashSet<>(); public QuarkusIO fetchQuarkusIo() { CompletableFuture main = null; Map> localized = new LinkedHashMap<>(); try (SimpleExecutor executor = new SimpleExecutor(fetchingConfig.parallelism())) { - main = executor.submit(() -> fetchQuarkusIoSite("quarkus.io", quarkusIOConfig.gitUri(), MAIN)); + main = executor.submit(() -> fetchQuarkusIoSite("quarkus.io", quarkusIOConfig.gitUri(), QuarkusIO.MAIN_BRANCHES)); for (Map.Entry entry : sortMap(quarkusIOConfig.localized()).entrySet()) { var language = entry.getKey(); var config = entry.getValue(); localized.put(language, - executor.submit( - () -> fetchQuarkusIoSite(language.code + ".quarkus.io", config.gitUri(), LOCALIZED))); + executor.submit(() -> fetchQuarkusIoSite(language.code + ".quarkus.io", config.gitUri(), + QuarkusIO.LOCALIZED_BRANCHES))); } executor.waitForSuccessOrThrow(fetchingConfig.timeout()); // If we get here, all tasks succeeded. @@ -70,13 +66,13 @@ public QuarkusIO fetchQuarkusIo() { } } - private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branches branches) { - URI requestedGitUri = gitUri; - CloseableDirectory repoDir = null; + private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, GitCloneDirectory.Branches branches) { + CloseableDirectory tempDir = null; + GitCloneDirectory cloneDir = null; try { - GitCloneDirectory.GitDirectoryDetails repository = repositories.get(gitUri); - if (repository != null) { - return repository.pull(branches); + GitCloneDirectory.Details details = detailsCache.get(gitUri); + if (details != null) { + return details.openAndUpdate(); } if (LaunchMode.DEVELOPMENT.equals(LaunchMode.current())) { @@ -84,37 +80,33 @@ private GitCloneDirectory fetchQuarkusIoSite(String siteName, URI gitUri, Branch Log.warnf("Unzipping '%s': this application is most likely indexing only a sample of %s." + " See README to index the full website.", gitUri, siteName); - repoDir = CloseableDirectory.temp(siteName); - unzip(Path.of(gitUri), repoDir.path()); + tempDir = CloseableDirectory.temp(siteName); + tempDirectories.add(tempDir); + unzip(Path.of(gitUri), tempDir.path()); + cloneDir = GitCloneDirectory.openAndUpdate(tempDir.path(), branches); } else if (isFile(gitUri)) { Log.infof("Using the git repository '%s' as-is without cloning to speed up indexing of %s.", gitUri, siteName); // In dev mode, we want to skip cloning when possible, to make things quicker. - repoDir = CloseableDirectory.of(Paths.get(gitUri)); + cloneDir = GitCloneDirectory.openAndUpdate(Path.of(gitUri), branches); } } - boolean requiresCloning; - if (repoDir == null) { + if (cloneDir == null) { // We always end up here in prod and tests. // That's fine, because prod will always use remote (http/git) git URIs anyway, // never local ones (file). - repoDir = CloseableDirectory.temp(siteName); - requiresCloning = true; - } else { - // We may end up here, but only in dev mode - requiresCloning = false; + // We may skip it in dev mode though. + tempDir = CloseableDirectory.temp(siteName); + tempDirectories.add(tempDir); + cloneDir = GitCloneDirectory.clone(gitUri, tempDir.path(), branches); } - closeableDirectories.add(repoDir); + detailsCache.put(gitUri, cloneDir.details()); - repository = new GitCloneDirectory.GitDirectoryDetails(repoDir.path(), branches.pages()); - repositories.put(requestedGitUri, repository); - - // If we have a local repository -- open it, and then pull the changes, clone it otherwise: - return requiresCloning ? repository.clone(gitUri, branches) : repository.pull(branches); + return cloneDir; } catch (RuntimeException | IOException e) { - new SuppressingCloser(e).push(repoDir); + new SuppressingCloser(e).push(tempDir).push(cloneDir); throw new IllegalStateException("Failed to fetch '%s': %s".formatted(siteName, e.getMessage()), e); } } @@ -138,20 +130,11 @@ private static boolean isZip(URI uri) { @PreDestroy public void cleanupTemporaryFolders() { try (Closer closer = new Closer<>()) { - closer.pushAll(CloseableDirectory::close, closeableDirectories); + closer.pushAll(CloseableDirectory::close, tempDirectories); } catch (Exception e) { throw new IllegalStateException( - "Failed to close directories '%s': %s".formatted(closeableDirectories, e.getMessage()), e); + "Failed to close directories '%s': %s".formatted(tempDirectories, e.getMessage()), e); } } - public record Branches(String sources, String pages) { - public List asRefList() { - return List.of("refs/heads/" + sources, "refs/heads/" + pages); - } - - public String[] asRefArray() { - return asRefList().toArray(String[]::new); - } - } } diff --git a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java index 277688d3..a537f07f 100644 --- a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java +++ b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java @@ -43,10 +43,10 @@ public class QuarkusIO implements AutoCloseable { public static final String QUARKUS_ORIGIN = "quarkus"; private static final String QUARKIVERSE_ORIGIN = "quarkiverse"; - public static final String SOURCE_BRANCH = "develop"; - public static final String PAGES_BRANCH = "master"; - public static final String LOCALIZED_SOURCE_BRANCH = "main"; - public static final String LOCALIZED_PAGES_BRANCH = "docs"; + public static final GitCloneDirectory.Branches MAIN_BRANCHES = new GitCloneDirectory.Branches( + "develop", "master"); + public static final GitCloneDirectory.Branches LOCALIZED_BRANCHES = new GitCloneDirectory.Branches( + "main", "docs"); public static URI httpUrl(URI urlBase, String version, String name) { return urlBase.resolve(httpPath(version, name)); diff --git a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java index 25ddcb6d..ae8ef0d9 100644 --- a/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java +++ b/src/main/java/io/quarkus/search/app/util/GitCloneDirectory.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Set; -import io.quarkus.search.app.fetching.FetchingService; import io.quarkus.search.app.fetching.LoggerProgressMonitor; import io.quarkus.logging.Log; @@ -23,43 +22,78 @@ public class GitCloneDirectory implements Closeable { + public static GitCloneDirectory clone(URI gitUri, Path directory, Branches branches) { + var details = new Details(directory, branches); + Log.infof("Cloning into '%s' from '%s'.", directory, gitUri); + Git git = null; + String remoteName = ORDERED_REMOTES.get(0); + try { + git = Git.cloneRepository() + .setURI(gitUri.toString()) + .setDirectory(directory.toFile()) + .setRemote(remoteName) + .setDepth(1) + .setNoTags() + .setBranch(branches.sources()) + .setBranchesToClone(branches.asRefList()) + .setProgressMonitor(LoggerProgressMonitor.create(log, "Cloning " + gitUri + ": ")) + // Unfortunately sparse checkouts are not supported: https://www.eclipse.org/forums/index.php/t/1094825/ + .call(); + return new GitCloneDirectory(git, details, remoteName); + } catch (RuntimeException | GitAPIException e) { + new SuppressingCloser(e).push(git); + throw new IllegalStateException( + "Failed to clone git repository into '%s' from '%s': %s".formatted(directory, gitUri, e.getMessage()), + e); + } + } + + public static GitCloneDirectory openAndUpdate(Path directory, Branches branches) { + var details = new Details(directory, branches); + return details.openAndUpdate(); + } + private static final Logger log = Logger.getLogger(GitCloneDirectory.class); private static final List ORDERED_REMOTES = Arrays.asList("upstream", "origin"); private final Git git; - private final GitDirectoryDetails directory; + private final Details details; private final String remoteName; private RevTree pagesTree; - public GitCloneDirectory(Git git, GitDirectoryDetails directory, String remoteName) { + public GitCloneDirectory(Git git, Details details, String remoteName) { this.git = git; - this.directory = directory; + this.details = details; this.remoteName = remoteName; } + public Details details() { + return details; + } + public Git git() { return git; } public Path resolve(String other) { - return directory.directory().resolve(other); + return details.directory().resolve(other); } - public GitDirectoryDetails directory() { - return directory; + public Details directory() { + return details; } public Path resolve(Path other) { - return directory.directory().resolve(other); + return details.directory().resolve(other); } public RevTree pagesTree() { if (this.pagesTree == null) { try { this.pagesTree = GitUtils.firstExistingRevTree(git.getRepository(), - (remoteName == null ? "" : remoteName + "/") + directory.pagesBranch()); + (remoteName == null ? "" : remoteName + "/") + details.branches.pages()); } catch (IOException e) { - throw new RuntimeException("Unable to locate pages branch: " + directory.pagesBranch(), e); + throw new RuntimeException("Unable to locate pages branch: " + details.branches.pages(), e); } } return pagesTree; @@ -77,13 +111,13 @@ public void close() throws IOException { public String toString() { return "GitCloneDirectory{" + "git=" + git + - ", directory=" + directory + + ", directory=" + details + '}'; } - public record GitDirectoryDetails(Path directory, String pagesBranch) { - public GitCloneDirectory pull(FetchingService.Branches branches) { - Log.infof("Pulling changes for '%s'.", directory); + public record Details(Path directory, Branches branches) { + public GitCloneDirectory openAndUpdate() { + Log.infof("Opening and updating '%s'.", directory); Git git = null; try { git = Git.open(directory.toFile()); @@ -98,7 +132,7 @@ public GitCloneDirectory pull(FetchingService.Branches branches) { } String remoteName = inferRemoteName(git); if (remoteName != null) { - pull(git, branches, remoteName); + update(git, remoteName); } // Else there's nowhere to pull from, so we don't even try. return new GitCloneDirectory(git, this, remoteName); @@ -109,7 +143,7 @@ public GitCloneDirectory pull(FetchingService.Branches branches) { } } - private void pull(Git git, FetchingService.Branches branches, String remoteName) { + private void update(Git git, String remoteName) { try { // fetch remote branches to make sure we'll use up-to-date data git.fetch() @@ -151,30 +185,15 @@ private String inferRemoteName(Git git) { // but let's give it one more chance and just pick any of the remotes at random. return remotes.iterator().next(); } + } - public GitCloneDirectory clone(URI gitUri, FetchingService.Branches branches) { - Log.infof("Cloning into '%s' from '%s'.", directory, gitUri); - Git git = null; - String remoteName = ORDERED_REMOTES.get(0); - try { - git = Git.cloneRepository() - .setURI(gitUri.toString()) - .setDirectory(directory.toFile()) - .setRemote(remoteName) - .setDepth(1) - .setNoTags() - .setBranch(branches.sources()) - .setBranchesToClone(branches.asRefList()) - .setProgressMonitor(LoggerProgressMonitor.create(log, "Cloning " + gitUri + ": ")) - // Unfortunately sparse checkouts are not supported: https://www.eclipse.org/forums/index.php/t/1094825/ - .call(); - return new GitCloneDirectory(git, this, remoteName); - } catch (RuntimeException | GitAPIException e) { - new SuppressingCloser(e).push(git); - throw new IllegalStateException( - "Failed to clone git repository into '%s' from '%s': %s".formatted(directory, gitUri, e.getMessage()), - e); - } + public record Branches(String sources, String pages) { + public List asRefList() { + return List.of("refs/heads/" + sources, "refs/heads/" + pages); + } + + public String[] asRefArray() { + return asRefList().toArray(String[]::new); } } } diff --git a/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java b/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java index 425eb6d1..a0dfd129 100644 --- a/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java +++ b/src/test/java/io/quarkus/search/app/fetching/FetchingServiceTest.java @@ -62,7 +62,7 @@ static void initOrigin() throws IOException, GitAPIException { Path guide1HtmlToFetch = sourceRepoPath.resolve("guides/" + FETCHED_GUIDE_1_NAME + ".html"); Path guide2HtmlToFetch = sourceRepoPath.resolve("version/2.7/guides/" + FETCHED_GUIDE_2_NAME + ".html"); try (Git git = Git.init().setDirectory(sourceRepoPath.toFile()) - .setInitialBranch(QuarkusIO.PAGES_BRANCH).call()) { + .setInitialBranch(QuarkusIO.MAIN_BRANCHES.pages()).call()) { GitTestUtils.cleanGitUserConfig(); RevCommit initialCommit = git.commit().setMessage("Initial commit") @@ -81,7 +81,7 @@ static void initOrigin() throws IOException, GitAPIException { git.commit().setMessage("Pages second commit").call(); git.checkout() - .setName(QuarkusIO.SOURCE_BRANCH) + .setName(QuarkusIO.MAIN_BRANCHES.sources()) .setCreateBranch(true) .setStartPoint(initialCommit) .call(); @@ -105,7 +105,7 @@ static void initOrigin() throws IOException, GitAPIException { Path localizedGuide2HtmlToFetch = localizedSourceRepoPath .resolve("docs/version/2.7/guides/" + FETCHED_GUIDE_2_NAME + ".html"); try (Git git = Git.init().setDirectory(localizedSourceRepoPath.toFile()) - .setInitialBranch(QuarkusIO.LOCALIZED_PAGES_BRANCH).call()) { + .setInitialBranch(QuarkusIO.LOCALIZED_BRANCHES.pages()).call()) { GitTestUtils.cleanGitUserConfig(); RevCommit initialCommit = git.commit().setMessage("Initial commit") @@ -124,7 +124,7 @@ static void initOrigin() throws IOException, GitAPIException { git.commit().setMessage("Pages second commit").call(); git.checkout() - .setName(QuarkusIO.LOCALIZED_SOURCE_BRANCH) + .setName(QuarkusIO.LOCALIZED_BRANCHES.sources()) .setCreateBranch(true) .setStartPoint(initialCommit) .call(); @@ -147,12 +147,12 @@ private void updateMainRepository() throws IOException, GitAPIException { try (Git git = Git.open(sourceRepoPath.toFile())) { GitTestUtils.cleanGitUserConfig(); - git.checkout().setName(QuarkusIO.PAGES_BRANCH).call(); + git.checkout().setName(QuarkusIO.MAIN_BRANCHES.pages()).call(); Files.writeString(guide1HtmlToFetch, FETCHED_GUIDE_1_CONTENT_HTML_UPDATED); git.add().addFilepattern(".").call(); git.commit().setMessage("Pages updated commit").call(); - git.checkout().setName(QuarkusIO.SOURCE_BRANCH).call(); + git.checkout().setName(QuarkusIO.MAIN_BRANCHES.sources()).call(); Files.writeString(metadata1ToFetch, METADATA_YAML_UPDATED); git.add().addFilepattern(".").call(); diff --git a/src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java b/src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java index aa6a422c..da447d6c 100644 --- a/src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java +++ b/src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java @@ -31,6 +31,7 @@ import io.quarkus.search.app.quarkusio.QuarkusIO; import io.quarkus.search.app.util.CloseableDirectory; import io.quarkus.search.app.util.FileUtils; +import io.quarkus.search.app.util.GitCloneDirectory; import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.common.QuarkusTestResourceConfigurableLifecycleManager; @@ -85,8 +86,7 @@ public static void main(String[] args) throws IOException { List remotes = Arrays.asList("upstream", "origin", null); try (CloseableDirectory copyRootDir = CloseableDirectory.temp("quarkusio-sample-building")) { - copy(originalPath, remotes, copyRootDir.path(), new AllFilterDefinition(), QuarkusIO.PAGES_BRANCH, - QuarkusIO.SOURCE_BRANCH, true); + copy(originalPath, remotes, copyRootDir.path(), new AllFilterDefinition(), QuarkusIO.MAIN_BRANCHES, true); Path sampleAbsolutePath = testResourcesSamplePath(); Files.deleteIfExists(sampleAbsolutePath); @@ -102,7 +102,7 @@ public static void main(String[] args) throws IOException { try (CloseableDirectory copyRootDir = CloseableDirectory.temp("quarkusio-sample-building")) { copy(originalLocalizedPath, remotes, copyRootDir.path(), new AllLocalizedFilterDefinition(language), - QuarkusIO.LOCALIZED_PAGES_BRANCH, QuarkusIO.LOCALIZED_SOURCE_BRANCH, false); + QuarkusIO.LOCALIZED_BRANCHES, false); Path sampleAbsolutePath = testResourcesSamplePath(language); Files.deleteIfExists(sampleAbsolutePath); @@ -112,23 +112,21 @@ public static void main(String[] args) throws IOException { } public static CloseableDirectory createFromTestResourcesSample(FilterDefinition filterDef) { - return createFromTestResourcesSample(filterDef, testResourcesSamplePath(), QuarkusIO.PAGES_BRANCH, - QuarkusIO.SOURCE_BRANCH, true); + return createFromTestResourcesSample(filterDef, testResourcesSamplePath(), QuarkusIO.MAIN_BRANCHES, true); } public static CloseableDirectory createFromTestResourcesLocalizedSample(Language language, FilterDefinition filterDef) { - return createFromTestResourcesSample(filterDef, testResourcesSamplePath(language), QuarkusIO.LOCALIZED_PAGES_BRANCH, - QuarkusIO.LOCALIZED_SOURCE_BRANCH, false); + return createFromTestResourcesSample(filterDef, testResourcesSamplePath(language), QuarkusIO.LOCALIZED_BRANCHES, false); } - private static CloseableDirectory createFromTestResourcesSample(FilterDefinition filterDef, Path path, String pagesBranch, - String sourceBranch, boolean failOnMissing) { + private static CloseableDirectory createFromTestResourcesSample(FilterDefinition filterDef, Path path, + GitCloneDirectory.Branches branches, boolean failOnMissing) { CloseableDirectory copyRootDir = null; try (CloseableDirectory unzippedQuarkusIoSample = CloseableDirectory.temp("quarkusio-sample-unzipped")) { FileUtils.unzip(path, unzippedQuarkusIoSample.path()); copyRootDir = CloseableDirectory.temp(filterDef.toString()); copy(unzippedQuarkusIoSample.path(), Collections.singletonList(null), - copyRootDir.path(), filterDef, pagesBranch, sourceBranch, failOnMissing); + copyRootDir.path(), filterDef, branches, failOnMissing); return copyRootDir; } catch (RuntimeException | IOException e) { new SuppressingCloser(e).push(copyRootDir); @@ -138,7 +136,7 @@ private static CloseableDirectory createFromTestResourcesSample(FilterDefinition } public static void copy(Path quarkusIoLocalPath, List originalRemotes, - Path copyRootPath, FilterDefinition filterDef, String pagesBranch, String sourceBranch, boolean failOnMissing) { + Path copyRootPath, FilterDefinition filterDef, GitCloneDirectory.Branches branches, boolean failOnMissing) { try (Git originalGit = Git.open(quarkusIoLocalPath.toFile())) { GitTestUtils.cleanGitUserConfig(); @@ -150,7 +148,7 @@ public static void copy(Path quarkusIoLocalPath, List originalRemotes, throw new IllegalStateException("No path to copy"); } - try (Git copyGit = Git.init().setInitialBranch(pagesBranch) + try (Git copyGit = Git.init().setInitialBranch(branches.pages()) .setDirectory(copyRootPath.toFile()).call()) { GitTestUtils.cleanGitUserConfig(); @@ -158,16 +156,16 @@ public static void copy(Path quarkusIoLocalPath, List originalRemotes, .setAllowEmpty(true) .call(); - copyIfNecessary(quarkusIoLocalPath, originalRepo, originalRemotes, pagesBranch, + copyIfNecessary(quarkusIoLocalPath, originalRepo, originalRemotes, branches.pages(), copyRootPath, copyGit, collector.pagesCopyPathToOriginalPath, failOnMissing); copyGit.checkout() - .setName(sourceBranch) + .setName(branches.sources()) .setCreateBranch(true) .setStartPoint(initialCommit) .call(); - copyIfNecessary(quarkusIoLocalPath, originalRepo, originalRemotes, sourceBranch, + copyIfNecessary(quarkusIoLocalPath, originalRepo, originalRemotes, branches.sources(), copyRootPath, copyGit, collector.sourceCopyPathToOriginalPath, failOnMissing); From fddfc2efd81fb65115b845e28766e53d779041a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Wed, 10 Jan 2024 18:25:07 +0100 Subject: [PATCH 8/8] Use thread-safe structures in FetchingService Because in theory they could be used from multiple threads in parallel. --- .../io/quarkus/search/app/fetching/FetchingService.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java index cc36e026..e41c5171 100644 --- a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java +++ b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java @@ -5,12 +5,11 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Path; -import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import jakarta.annotation.PreDestroy; @@ -30,6 +29,8 @@ import org.hibernate.search.util.common.impl.Closer; import org.hibernate.search.util.common.impl.SuppressingCloser; +import io.vertx.core.impl.ConcurrentHashSet; + @ApplicationScoped public class FetchingService { @@ -39,8 +40,8 @@ public class FetchingService { @Inject QuarkusIOConfig quarkusIOConfig; - private final Map detailsCache = new HashMap<>(); - private final Set tempDirectories = new HashSet<>(); + private final Map detailsCache = new ConcurrentHashMap<>(); + private final Set tempDirectories = new ConcurrentHashSet<>(); public QuarkusIO fetchQuarkusIo() { CompletableFuture main = null;