Skip to content

Commit

Permalink
ref(proguard): Use existing chunk uploading logic
Browse files Browse the repository at this point in the history
Use the existing chunk uploading logic, which is also used for debug files, in the Proguard chunk uploading code path
  • Loading branch information
szokeasaurusrex committed Dec 16, 2024
1 parent 3c8f73c commit aa015da
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 54 deletions.
2 changes: 1 addition & 1 deletion src/commands/upload_proguard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ pub fn execute(matches: &ArgMatches) -> Result<()> {
})
})?;

proguard::chunk_upload(&mappings, &chunk_upload_options, &org, &project)?;
proguard::chunk_upload(&mappings, chunk_upload_options, &org, &project)?;
} else {
if mappings.is_empty() && matches.get_flag("require_one") {
println!();
Expand Down
2 changes: 1 addition & 1 deletion src/utils/chunks/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ mod upload;

pub use options::ChunkOptions;
pub use types::{Assemblable, Chunked, MissingObjectsInfo};
pub use upload::upload_chunked_objects;
pub use upload::{upload_chunked_objects, ChunkOptions};

use std::sync::Arc;
use std::time::Duration;
Expand Down
69 changes: 68 additions & 1 deletion src/utils/chunks/upload.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
use std::{collections::BTreeMap, fmt::Display, thread, time::Instant};
use std::collections::BTreeMap;
use std::fmt::Display;
use std::time::{Duration, Instant};
use std::{cmp, thread};

use anyhow::Result;
use indicatif::ProgressStyle;
Expand Down Expand Up @@ -321,3 +324,67 @@ fn render_detail(detail: Option<&str>, fallback: Option<&str>) {
}
}
}

/// A struct representing options for chunk uploads. This implementation
/// of `ChunkOptions` should be general enough to be used for any chunk upload.
pub struct GenericChunkOptions<'a> {
server_options: ChunkServerOptions,
org: &'a str,
project: &'a str,

/// The maximum wait time for the upload to complete.
/// If set to zero, we do not wait for the upload to complete.
/// If the server_options.max_wait is set to a smaller nonzero value,
/// we use that value instead.
max_wait: Duration,
}

impl<'a> GenericChunkOptions<'a> {
/// Create a new `GenericChunkOptions` instance, with the provided server options,
/// organization, and project, and no wait time for the upload to complete.
pub fn new(server_options: ChunkServerOptions, org: &'a str, project: &'a str) -> Self {
Self {
server_options,
org,
project,
max_wait: Duration::ZERO,
}
}

/// Set the maximum wait time for the assembly to complete.
pub fn set_max_wait(&mut self, max_wait: Duration) {
self.max_wait = max_wait;
}
}

impl ChunkOptions for GenericChunkOptions<'_> {
fn should_strip_debug_ids(&self) -> bool {
self.server_options.should_strip_debug_ids()
}

fn org(&self) -> &str {
self.org
}

fn project(&self) -> &str {
self.project
}

fn should_wait(&self) -> bool {
!self.max_wait().is_zero()
}

fn max_wait(&self) -> Duration {
// If the server specifies a max wait time (indicated by a nonzero value),
// we use the minimum of the user-specified max wait time and the server's
// max wait time.
match self.server_options.max_wait {
0 => self.max_wait,
server_max_wait => cmp::min(self.max_wait, Duration::from_secs(server_max_wait)),
}
}

fn server_options(&self) -> &ChunkServerOptions {
&self.server_options
}
}
7 changes: 7 additions & 0 deletions src/utils/proguard/mapping.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use std::borrow::Cow;
use std::fmt::{Display, Formatter, Result as FmtResult};

use symbolic::common::{ByteView, DebugId};
use thiserror::Error;
Expand Down Expand Up @@ -76,3 +77,9 @@ impl Assemblable for ProguardMapping<'_> {
None
}
}

impl Display for ProguardMapping<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
write!(f, "{} (Proguard mapping)", self.uuid)
}
}
63 changes: 12 additions & 51 deletions src/utils/proguard/upload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,15 @@
//! Proguard mappings, while we work on a more permanent solution, which will
//! work for all different types of debug files.
use std::thread;
use std::time::{Duration, Instant};
use std::time::Duration;

use anyhow::Result;
use indicatif::ProgressStyle;

use crate::api::{Api, ChunkServerOptions, ChunkedFileState};
use crate::utils::chunks;
use crate::utils::chunks::Chunked;
use crate::api::ChunkServerOptions;
use crate::utils::chunks::GenericChunkOptions;
use crate::utils::chunks::{ChunkOptions, Chunked};
use crate::utils::proguard::ProguardMapping;

/// How often to poll the server for the status of the assembled mappings.
const ASSEMBLE_POLL_INTERVAL: Duration = Duration::from_secs(1);

/// How long to wait for the server to assemble the mappings before giving up.
// 120 seconds was chosen somewhat arbitrarily, but in my testing, assembly
// usually was almost instantaneous, so this should probably be enough time.
Expand All @@ -28,7 +23,7 @@ const ASSEMBLE_POLL_TIMEOUT: Duration = Duration::from_secs(120);
/// Returns an error if the mappings fail to assemble, or if the timeout is reached.
pub fn chunk_upload(
mappings: &[ProguardMapping<'_>],
chunk_upload_options: &ChunkServerOptions,
chunk_upload_options: ChunkServerOptions,
org: &str,
project: &str,
) -> Result<()> {
Expand All @@ -37,48 +32,14 @@ pub fn chunk_upload(
.map(|mapping| Chunked::from(mapping, chunk_upload_options.chunk_size as usize))
.collect::<Result<Vec<_>>>()?;

let progress_style = ProgressStyle::default_bar().template(
"Uploading Proguard mappings...\
\n{wide_bar} {bytes}/{total_bytes} ({eta})",
);

let chunks = chunked_mappings
.iter()
.flat_map(|mapping| mapping.iter_chunks());

chunks::upload_chunks(
&chunks.collect::<Vec<_>>(),
chunk_upload_options,
progress_style,
)?;
let mut options = GenericChunkOptions::new(chunk_upload_options, org, project);
options.set_max_wait(ASSEMBLE_POLL_TIMEOUT);

println!("Waiting for server to assemble uploaded mappings...");
let (_, has_processing_errors) = options.upload_chunked_objects(&chunked_mappings)?;

let assemble_request = chunked_mappings.iter().collect();
let start = Instant::now();
while Instant::now().duration_since(start) < ASSEMBLE_POLL_TIMEOUT {
let all_assembled = Api::current()
.authenticated()?
.assemble_difs(org, project, &assemble_request)?
.values()
.map(|response| match response.state {
ChunkedFileState::Error => anyhow::bail!("Error: {response:?}"),
ChunkedFileState::NotFound => anyhow::bail!("File not found: {response:?}"),
ChunkedFileState::Ok | ChunkedFileState::Created | ChunkedFileState::Assembling => {
Ok(response)
}
})
.collect::<Result<Vec<_>>>()?
.iter()
.all(|response| matches!(response.state, ChunkedFileState::Ok));

if all_assembled {
println!("Server finished assembling mappings.");
return Ok(());
}

thread::sleep(ASSEMBLE_POLL_INTERVAL);
if has_processing_errors {
Err(anyhow::anyhow!("Some symbols did not process correctly"))
} else {
Ok(())
}

anyhow::bail!("Timed out waiting for server to assemble uploaded mappings.")
}

0 comments on commit aa015da

Please sign in to comment.