From c5293fc7f9616c3b5fc1d00849b3d789c42f8388 Mon Sep 17 00:00:00 2001
From: Alex Butler <alexheretic@gmail.com>
Date: Wed, 14 Feb 2024 12:44:44 +0000
Subject: [PATCH] Add PARALLEL_PACKAGE_FILES_GETS = 32

---
 src/providers/gitlab.rs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs
index e3a3209..31571c3 100644
--- a/src/providers/gitlab.rs
+++ b/src/providers/gitlab.rs
@@ -14,9 +14,13 @@ use reqwest::{header, Certificate};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use time::{Duration, OffsetDateTime};
+use tokio::sync::Semaphore;
 use tracing::{debug, info_span, instrument, Instrument};
 use url::Url;
 
+/// Number of `package_files` GETs to do in parallel.
+const PARALLEL_PACKAGE_FILES_GETS: usize = 32;
+
 pub struct Gitlab {
     client: reqwest::Client,
     base_url: Url,
@@ -174,6 +178,7 @@ impl super::PackageProvider for Gitlab {
             uri
         });
 
+        let fetch_concurrency = Semaphore::new(PARALLEL_PACKAGE_FILES_GETS);
         let futures = FuturesUnordered::new();
 
         while let Some(uri) = next_uri.take() {
@@ -204,9 +209,12 @@ impl super::PackageProvider for Gitlab {
             for release in res {
                 let this = Arc::clone(&self);
                 let do_as = Arc::clone(do_as);
+                let fetch_concurrency = &fetch_concurrency;
 
                 futures.push(
                     async move {
+                        let _guard = fetch_concurrency.acquire().await?;
+
                         let (project, package) = {
                             let mut splitter = release.links.web_path.splitn(2, "/-/packages/");
                             match (splitter.next(), splitter.next()) {