diff --git a/lib/src/default_working_copy_store.rs b/lib/src/default_working_copy_store.rs new file mode 100644 index 0000000000..91b7c236a6 --- /dev/null +++ b/lib/src/default_working_copy_store.rs @@ -0,0 +1,330 @@ +// Copyright 2023 The Jujutsu Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This file contains the default implementation of the `WorkingCopyStore` for both the Git and +//! native Backend. It stores the working copies in the `.jj/run/default` path as directories. +use std::any::Any; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::OnceLock; + +use itertools::Itertools; + +use crate::backend::MergedTreeId; +use crate::commit::{self, Commit}; +use crate::local_working_copy::TreeState; +use crate::object_id::ObjectId; +use crate::repo::Repo; +use crate::revset::{RevsetExpression, RevsetIteratorExt}; +use crate::store::Store; +use crate::working_copy_store::{CachedWorkingCopy, WorkingCopyStore, WorkingCopyStoreError}; + +/// A thin wrapper over a `TreeState` for now. +// TODO: Move this to a LocalWorkingCopy instead of using just the TreeState. +#[derive(Clone)] +struct StoredWorkingCopy { + /// The actual commit which owns the associated [`TreeState`]. + commit: Commit, + /// Current state of the associated [`WorkingCopy`]. + state: Arc, + /// The output path for tools, which do not specify a location. Like C(++) Compilers, scripts and more. + /// It also contains the respective output stream, so stderr and stdout which was redirected for this commit. + output_path: PathBuf, + /// Path to the associated working copy. + working_copy_path: PathBuf, + /// Path to the associated tree state. + state_path: PathBuf, + /// Is this working-copy in use? + pub(crate) is_used: bool, +} + +impl StoredWorkingCopy { + /// Set up a `StoredWorkingCopy`. It's assumed that all paths exist on disk. + fn create( + store: Arc, + commit: Commit, + output_path: PathBuf, + working_copy_path: PathBuf, + state_path: PathBuf, + ) -> Self { + // Load the tree for our commit. + let state = Arc::new( + TreeState::load(store, working_copy_path.clone(), state_path.clone()).unwrap(), + ); + Self { + commit, + state, + output_path, + working_copy_path, + state_path, + is_used: false, + } + } + + /// Replace the currently cached working-copy and it's tree with the tree from `commit`. + /// Automatically marks it as used. + fn replace_with(&mut self, commit: &Commit) -> Result { + let Self { + commit: _, + ref mut state, + output_path, + working_copy_path, + state_path, + is_used: _, + } = self; + state.check_out(&commit.tree()?).map_err(|e| { + WorkingCopyStoreError::TreeUpdate(format!( + "failed to update the local working-copy with {e:?}" + )) + })?; + + Ok(Self { + commit: commit.clone(), + state: state.clone(), + output_path: output_path.to_path_buf(), + working_copy_path: working_copy_path.to_path_buf(), + state_path: state_path.to_path_buf(), + is_used: true, + }) + } +} + +/// The default [`WorkingCopyStore`] for both the Git and native backend. +// TODO: Offload the creation of working copy directories onto a threadpool. +#[derive(Default)] +pub struct DefaultWorkingCopyStore { + /// Where the working copies are stored, in this case `.jj/run/default/` + stored_paths: PathBuf, + /// All managed working copies. + stored_working_copies: Vec, + /// The store which owns this and all other backend related stuff. It gets set during the first + /// creation of the managed working copies. + store: OnceLock>, +} + +/// Creates the required directories for a StoredWorkingCopy. +/// Returns a tuple of (`output_dir`, `working_copy` and `state`). +fn create_working_copy_paths( + path: &PathBuf, +) -> Result<(PathBuf, PathBuf, PathBuf), std::io::Error> { + let output = path.join("output"); + let working_copy = path.join("working_copy"); + let state = path.join("state"); + std::fs::create_dir(&output)?; + std::fs::create_dir(&working_copy)?; + std::fs::create_dir(&state)?; + Ok((output, working_copy, state)) +} + +/// Represent a `MergeTreeId` in a way that it may be used as a working-copy +/// name. This makes no stability guarantee, as the format may change at +/// any time. +fn to_wc_name(id: &MergedTreeId) -> String { + match id { + MergedTreeId::Legacy(tree_id) => tree_id.hex(), + MergedTreeId::Merge(tree_ids) => { + let ids = tree_ids + .map(|id| id.hex()) + .iter_mut() + .enumerate() + .map(|(i, s)| { + // Incredibly "smart" way to say, append "-" if the number is odd "+" + // otherwise. + if i & 1 != 0 { + s.push('-'); + } else { + s.push('+'); + } + s.to_owned() + }) + .collect_vec(); + let mut obfuscated: String = ids.concat(); + // `PATH_MAX` could be a problem for different operating systems, so truncate it. + if obfuscated.len() >= 255 { + obfuscated.truncate(200); + } + obfuscated + } + } +} + +impl DefaultWorkingCopyStore { + pub fn name() -> &'static str { + "default" + } + + pub fn init(dot_dir: &Path) -> Self { + let stored_paths = dot_dir.join(Self::name()); + // If the toplevel dir doesn't exist, create it. + if !stored_paths.exists() { + // TODO: correct error handling + std::fs::create_dir(stored_paths.clone()).expect("shouldn't fail"); + } + + Self { + stored_paths, + ..Default::default() + } + } + + pub fn load(dot_dir: &Path) -> Self { + Self::init(dot_dir) + } + + fn create_working_copies( + &mut self, + revisions: &[Commit], + ) -> Result>, std::io::Error> { + let store = revisions + .first() + .expect("revisions shouldn't be empty") + .store(); + // only set the store if we're a fresh call or a reload. + self.store.get_or_init(|| store.clone()); + let mut results: Vec> = Vec::new(); + // Use the tree id for a unique directory. + for rev in revisions { + let tree_id = to_wc_name(&rev.tree_id()); + let path: PathBuf = self.stored_paths.join(tree_id); + // Create a dir under `.jj/run/`. + std::fs::create_dir(&path)?; + // And the additional directories. + let (output, working_copy_path, state) = create_working_copy_paths(&path)?; + let cached_wc = StoredWorkingCopy::create( + store.clone(), + rev.clone(), + output, + working_copy_path, + state, + ); + let cached_clone = cached_wc.clone(); + self.stored_working_copies.push(cached_wc); + results.push(Box::new(cached_clone) as Box); + } + Ok(results) + } +} + +impl WorkingCopyStore for DefaultWorkingCopyStore { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &'static str { + Self::name() + } + + fn get_or_create_working_copies( + &self, + handle: Self::Handle, + repo: &dyn Repo, + revisions: Vec, + ) -> Result>, WorkingCopyStoreError> { + // This is the initial call for a Workspace, so just create working-copies. + if self.stored_working_copies.is_empty() { + return Ok(self.create_working_copies(&revisions)?); + } + assert!( + !self.stored_working_copies.is_empty(), + "we must have working copies after the first call" + ); + // If we already have some existing working copies, try to minimize pending work. + // This is done by finding the intersection of the existing and new commits and only + // creating the non-overlapping revisions. + let new_revision_ids = revisions.iter().map(|rev| rev.id().clone()).collect_vec(); + let contained_revisions = self + .stored_working_copies + .iter() + .map(|sc| sc.commit.id().clone()) + .collect_vec(); + let contained_revset = RevsetExpression::commits(contained_revisions); + // intersect the existing revisions with the newly requested revisions to see which need to + // be replaced. + let overlapping_commits_revset = + &contained_revset.intersection(&RevsetExpression::commits(new_revision_ids)); + let overlappping_commits: Vec = overlapping_commits_revset + .clone() + .evaluate_programmatic(repo)? + .iter() + .commits(self.store.get().unwrap()) + .try_collect()?; + // the new revisions which we need to create. + let new_revisions: Vec = overlapping_commits_revset + .minus(&contained_revset) + .evaluate_programmatic(repo)? + .iter() + .commits(self.store.get().unwrap()) + .try_collect()?; + + self.stored_working_copies + .iter_mut() + .filter(|sc| !overlappping_commits.contains(&sc.commit)) + // I don't know if this works. + .map(|sc| sc.replace_with(new_revisions.iter().next().unwrap())); + + // the caller is going to use the working-copies so mark them as that. + self.stored_working_copies + .iter_mut() + .map(|sc| sc.is_used = true); + + Ok(self + .stored_working_copies + .iter() + .map(|sc| Box::new(sc.clone()) as Box) + .collect_vec()) + } + + fn has_stores(&self) -> bool { + !self.stored_working_copies.is_empty() + } + + fn unused_stores(&self, handle: Self::Handle) -> usize { + handle.has_empty(&self) + } + + fn update_working_copies( + &self, + _repo: &dyn Repo, + replacements: Vec, + ) -> Result<(), WorkingCopyStoreError> { + // Find multiple unused working copies and replace them. + let mut old_wcs = handle + .stored_working_copies + .iter() + .filter(|sc| !sc.is_used) + .collect_vec(); + // TODO: is this correct? + old_wcs.iter_mut().map(|wc| { + wc.replace_with(replacements.iter().next().unwrap()) + .ok() + .unwrap() + }); + + Ok(()) + } + + fn update_single( + &self, + handle: Self::Handle, + new_commit: Commit, + ) -> Result<(), WorkingCopyStoreError> { + let old_wc: &mut StoredWorkingCopy = self + .stored_working_copies + .iter_mut() + .find(|sc| !sc.is_used) + .unwrap(); + old_wc.replace_with(&new_commit)?; + Ok(()) + } +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 149956ad2b..87b96c6681 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -35,6 +35,7 @@ pub mod conflicts; pub mod dag_walk; pub mod default_index; pub mod default_submodule_store; +pub mod default_working_copy_store; pub mod diff; pub mod dsl_util; pub mod extensions_map; @@ -90,4 +91,5 @@ pub mod tree_builder; pub mod union_find; pub mod view; pub mod working_copy; +pub mod working_copy_store; pub mod workspace; diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 1382dc9224..c1c14ace80 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -36,6 +36,7 @@ use crate::commit::{Commit, CommitByCommitterTimestamp}; use crate::commit_builder::CommitBuilder; use crate::default_index::{DefaultIndexStore, DefaultMutableIndex}; use crate::default_submodule_store::DefaultSubmoduleStore; +use crate::default_working_copy_store::DefaultWorkingCopyStore; use crate::file_util::{IoResultExt as _, PathError}; use crate::index::{ChangeIdIndex, Index, IndexStore, MutableIndex, ReadonlyIndex}; use crate::local_backend::LocalBackend; @@ -58,6 +59,7 @@ use crate::store::Store; use crate::submodule_store::SubmoduleStore; use crate::transaction::Transaction; use crate::view::View; +use crate::working_copy_store::WorkingCopyStore; use crate::{backend, dag_walk, op_store, revset}; pub trait Repo { @@ -71,6 +73,8 @@ pub trait Repo { fn submodule_store(&self) -> &Arc; + fn working_copy_store(&self) -> &Arc; + fn resolve_change_id(&self, change_id: &ChangeId) -> Option> { // Replace this if we added more efficient lookup method. let prefix = HexPrefix::from_bytes(change_id.as_bytes()); @@ -96,6 +100,8 @@ pub struct ReadonlyRepo { index_store: Arc, submodule_store: Arc, index: OnceCell>, + working_copy_store: Arc, + // Declared after `change_id_index` since it must outlive it on drop. change_id_index: OnceCell>, // TODO: This should eventually become part of the index and not be stored fully in memory. view: View, @@ -138,6 +144,11 @@ impl ReadonlyRepo { &|_settings, store_path| Box::new(DefaultSubmoduleStore::init(store_path)) } + pub fn default_working_copy_store_initializer() -> &'static WorkingCopyStoreInitializer<'static> + { + &|store_path| Box::new(DefaultWorkingCopyStore::init(store_path)) + } + #[allow(clippy::too_many_arguments)] pub fn init( user_settings: &UserSettings, @@ -148,6 +159,7 @@ impl ReadonlyRepo { op_heads_store_initializer: &OpHeadsStoreInitializer, index_store_initializer: &IndexStoreInitializer, submodule_store_initializer: &SubmoduleStoreInitializer, + working_copy_store_initializer: &WorkingCopyStoreInitializer, ) -> Result, RepoInitError> { let repo_path = repo_path.canonicalize().context(repo_path)?; @@ -189,6 +201,14 @@ impl ReadonlyRepo { .context(&submodule_store_type_path)?; let submodule_store = Arc::from(submodule_store); + let working_copy_store_path = repo_path.join("working_copy_store"); + fs::create_dir(&working_copy_store_path).context(&working_copy_store_path)?; + let working_copy_store = working_copy_store_initializer(&working_copy_store_path); + let working_copy_store_type_path = working_copy_store_path.join("type"); + fs::write(&working_copy_store_type_path, working_copy_store.name()) + .context(&working_copy_store_path)?; + let working_copy_store = Arc::from(working_copy_store); + let root_operation_data = op_store .read_operation(op_store.root_operation_id()) .expect("failed to read root operation"); @@ -209,6 +229,7 @@ impl ReadonlyRepo { index: OnceCell::new(), change_id_index: OnceCell::new(), view: root_view, + working_copy_store, submodule_store, }); let mut tx = repo.start_transaction(user_settings); @@ -227,6 +248,7 @@ impl ReadonlyRepo { op_heads_store: self.op_heads_store.clone(), index_store: self.index_store.clone(), submodule_store: self.submodule_store.clone(), + working_copy_store: self.working_copy_store.clone(), } } @@ -321,6 +343,10 @@ impl Repo for ReadonlyRepo { &self.submodule_store } + fn working_copy_store(&self) -> &Arc { + &self.working_copy_store + } + fn resolve_change_id_prefix(&self, prefix: &HexPrefix) -> PrefixResolution> { self.change_id_index().resolve_prefix(prefix) } @@ -338,6 +364,7 @@ pub type IndexStoreInitializer<'a> = dyn Fn(&UserSettings, &Path) -> Result, BackendInitError> + 'a; pub type SubmoduleStoreInitializer<'a> = dyn Fn(&UserSettings, &Path) -> Box + 'a; +pub type WorkingCopyStoreInitializer<'a> = dyn Fn(&Path) -> Box + 'a; type BackendFactory = Box Result, BackendLoadError>>; @@ -346,6 +373,7 @@ type OpHeadsStoreFactory = Box Box Result, BackendLoadError>>; type SubmoduleStoreFactory = Box Box>; +type WorkingCopyStoreFactory = Box Box>; pub fn merge_factories_map(base: &mut HashMap, ext: HashMap) { for (name, factory) in ext { @@ -366,6 +394,7 @@ pub struct StoreFactories { op_heads_store_factories: HashMap, index_store_factories: HashMap, submodule_store_factories: HashMap, + stored_working_copy_factories: HashMap, } impl Default for StoreFactories { @@ -420,6 +449,12 @@ impl Default for StoreFactories { Box::new(|_settings, store_path| Box::new(DefaultSubmoduleStore::load(store_path))), ); + // WorkingCopyStores + factories.add_working_copy_store( + DefaultWorkingCopyStore::name(), + Box::new(|_settings, store_path| Box::new(DefaultWorkingCopyStore::load(store_path))), + ); + factories } } @@ -450,6 +485,7 @@ impl StoreFactories { op_heads_store_factories: HashMap::new(), index_store_factories: HashMap::new(), submodule_store_factories: HashMap::new(), + stored_working_copy_factories: HashMap::new(), } } @@ -572,6 +608,31 @@ impl StoreFactories { Ok(submodule_store_factory(settings, store_path)) } + + pub fn add_working_copy_store(&mut self, name: &str, factory: WorkingCopyStoreFactory) { + self.stored_working_copy_factories + .insert(name.to_string(), factory); + } + + pub fn load_working_copy_store( + &self, + settings: &UserSettings, + store_path: &Path, + ) -> Result, StoreLoadError> { + // For compatibility with repos without a working_copy_store. + // TODO Delete default in TBD version + let working_copy_store_type = + read_store_type("working_copy_store", store_path.join("type"))?; + let working_copy_store_factory = self + .stored_working_copy_factories + .get(&working_copy_store_type) + .ok_or_else(|| StoreLoadError::UnsupportedType { + store: "working_copy_store", + store_type: working_copy_store_type.to_string(), + })?; + + Ok(working_copy_store_factory(settings, store_path)) + } } pub fn read_store_type( @@ -603,6 +664,7 @@ pub struct RepoLoader { op_heads_store: Arc, index_store: Arc, submodule_store: Arc, + working_copy_store: Arc, } impl RepoLoader { @@ -628,6 +690,11 @@ impl RepoLoader { store_factories .load_submodule_store(user_settings, &repo_path.join("submodule_store"))?, ); + let working_copy_store = Arc::from( + store_factories + .load_working_copy_store(user_settings, &repo_path.join("working_copy_store"))?, + ); + Ok(Self { repo_path: repo_path.to_path_buf(), repo_settings, @@ -636,6 +703,7 @@ impl RepoLoader { op_heads_store, index_store, submodule_store, + working_copy_store, }) } @@ -694,6 +762,7 @@ impl RepoLoader { index_store: self.index_store.clone(), submodule_store: self.submodule_store.clone(), index: OnceCell::with_value(index), + working_copy_store: self.working_copy_store.clone(), change_id_index: OnceCell::new(), view, }; @@ -727,6 +796,7 @@ impl RepoLoader { settings: self.repo_settings.clone(), index_store: self.index_store.clone(), submodule_store: self.submodule_store.clone(), + working_copy_store: self.working_copy_store.clone(), index: OnceCell::new(), change_id_index: OnceCell::new(), view, @@ -1716,15 +1786,19 @@ impl Repo for MutableRepo { self.index.as_index() } + fn submodule_store(&self) -> &Arc { + self.base_repo.submodule_store() + } + + fn working_copy_store(&self) -> &Arc { + self.base_repo.working_copy_store() + } + fn view(&self) -> &View { self.view .get_or_ensure_clean(|v| self.enforce_view_invariants(v)) } - fn submodule_store(&self) -> &Arc { - self.base_repo.submodule_store() - } - fn resolve_change_id_prefix(&self, prefix: &HexPrefix) -> PrefixResolution> { let change_id_index = self.index.change_id_index(&mut self.view().heads().iter()); change_id_index.resolve_prefix(prefix) diff --git a/lib/src/working_copy_store.rs b/lib/src/working_copy_store.rs new file mode 100644 index 0000000000..10cf55fe00 --- /dev/null +++ b/lib/src/working_copy_store.rs @@ -0,0 +1,87 @@ +// Copyright 2023 The Jujutsu Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This file contains the [`WorkingCopyStore`] interface which is used to cached working copies. +//! +//! These must be implemented for Virtual Filesystems such as [EdenFS] +//! to allow cheaper working copy materializations, they are used for the `jj run` +//! implementation. +//! +//! +//! [EdenFS]: www.github.com/facebook/sapling/main/blob/eden/fs + +use std::{any::Any, convert::Infallible, io}; + +use crate::{ + backend::{BackendError, CommitId}, + commit::Commit, + local_working_copy::LocalWorkingCopy, + repo::Repo, + revset::RevsetEvaluationError, +}; +use thiserror::Error; + +/// An Error from the Cache, which [`WorkingCopyStore`] represents. +#[derive(Debug, Error)] +pub enum WorkingCopyStoreError { + /// We failed to initialize something, the store or any underlying working-copies. + #[error("failed to initialize")] + Initialization(#[from] io::Error), + /// An error occured during a `CachedWorkingCopy` update. + #[error("could not update the working copy {0}")] + TreeUpdate(String), + /// If the backend failed internally. + #[error("backend failed internally")] + Backend(#[from] BackendError), + /// Any internal error, which shouldn't be propagated to the user. + // TODO: This ideally also should contain the `RevsetError`, as it purely is an implementation + // detail. + #[error("internal error")] + Internal(#[from] Infallible), + // The variant below shouldn't exist. + #[error("revset evaluation failed")] + Revset(#[from] RevsetEvaluationError), +} + +/// A `WorkingCopyStore` manages the working copies on disk for `jj run`. +/// It's an ideal extension point for an virtual filesystem, as they ease the creation of +/// working copies. +/// +/// The trait's design is similar to a database. Clients request a single or multiple working-copies +/// and the backend can coalesce the requests if needed. This allows an implementation to build +/// a global view of all actively used working-copies and where they are stored. +pub trait WorkingCopyStore: Send + Sync { + /// Return `self` as `Any` to allow trait upcasting. + fn as_any(&self) -> &dyn Any; + + /// The name of the backend, determines how it actually interacts with working copies. + fn name(&self) -> &str; + + /// Get existing or create `Stores` for `revisions`. + fn get_or_create_working_copies( + &self, + repo: &dyn Repo, + revisions: &[CommitId], + ) -> Result, WorkingCopyStoreError>; + + /// Update multiple stored working copies at once, akin to a sql update. + fn update_working_copies( + &self, + repo: &dyn Repo, + replacements: &[CommitId], + ) -> Result<(), WorkingCopyStoreError>; + + /// Update a single working-copy, determined by the backend. + fn update_single(&self, new_commit: CommitId) -> Result<(), WorkingCopyStoreError>; +} diff --git a/lib/src/workspace.rs b/lib/src/workspace.rs index 624a80d935..a79ea2dc84 100644 --- a/lib/src/workspace.rs +++ b/lib/src/workspace.rs @@ -32,7 +32,7 @@ use crate::op_store::{OperationId, WorkspaceId}; use crate::repo::{ read_store_type, BackendInitializer, CheckOutCommitError, IndexStoreInitializer, OpHeadsStoreInitializer, OpStoreInitializer, ReadonlyRepo, Repo, RepoInitError, RepoLoader, - StoreFactories, StoreLoadError, SubmoduleStoreInitializer, + StoreFactories, StoreLoadError, SubmoduleStoreInitializer, WorkingCopyStoreInitializer, }; use crate::settings::UserSettings; use crate::signing::{SignInitError, Signer}; @@ -250,6 +250,7 @@ impl Workspace { op_heads_store_initializer: &OpHeadsStoreInitializer, index_store_initializer: &IndexStoreInitializer, submodule_store_initializer: &SubmoduleStoreInitializer, + working_copy_store_initializer: &WorkingCopyStoreInitializer, working_copy_factory: &dyn WorkingCopyFactory, workspace_id: WorkspaceId, ) -> Result<(Self, Arc), WorkspaceInitError> { @@ -266,6 +267,7 @@ impl Workspace { op_heads_store_initializer, index_store_initializer, submodule_store_initializer, + working_copy_store_initializer, ) .map_err(|repo_init_err| match repo_init_err { RepoInitError::Backend(err) => WorkspaceInitError::Backend(err), @@ -304,6 +306,7 @@ impl Workspace { ReadonlyRepo::default_op_heads_store_initializer(), ReadonlyRepo::default_index_store_initializer(), ReadonlyRepo::default_submodule_store_initializer(), + ReadonlyRepo::default_working_copy_store_initializer(), &*default_working_copy_factory(), WorkspaceId::default(), )