From f553c4d779f06fdd6b77247296271b55cfc597ee Mon Sep 17 00:00:00 2001 From: Philip Metzger Date: Tue, 15 Aug 2023 16:42:24 +0200 Subject: [PATCH] lib: Add the WorkingCopyStore trait and a default implementation. In principle this trait is a layer above an actual VFS, but should be enough to start working with them. It's responsible for caching and managing working-copies which are stored locally or on a remote. The plan is that `jj run` will query it to request working copies, which it then uses. This checks the third checkmark in #1869. Progress on #1869 and #405 cc @martinvonz, @hooper, @kevincliao, @arxanas --- lib/src/default_working_copy_store.rs | 179 ++++++++++++++++++++++++++ lib/src/lib.rs | 2 + lib/src/repo.rs | 71 ++++++++++ lib/src/working_copy_store.rs | 43 +++++++ 4 files changed, 295 insertions(+) create mode 100644 lib/src/default_working_copy_store.rs create mode 100644 lib/src/working_copy_store.rs diff --git a/lib/src/default_working_copy_store.rs b/lib/src/default_working_copy_store.rs new file mode 100644 index 0000000000..3591835f57 --- /dev/null +++ b/lib/src/default_working_copy_store.rs @@ -0,0 +1,179 @@ +// Copyright 2023 The Jujutsu Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This file contains the default implementation of the `WorkingCopyStore` for both the Git and +//! Native Backend. +//! +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use itertools::Itertools; + +use crate::commit::Commit; +use crate::local_working_copy::TreeState; +use crate::store::Store; +use crate::working_copy_store::{CachedWorkingCopy, WorkingCopyStore}; + +/// A thin wrapper over a `TreeState` for now. +#[derive(Debug)] +struct StoredWorkingCopy { + /// Current state of the associated [`WorkingCopy`]. + state: TreeState, + /// The output path for tools, which do not specify a location. + /// Like C(++) Compilers, scripts and more. + /// TODO: Is this necessary? + output_path: PathBuf, + /// Path to the associated working copy. + working_copy_path: PathBuf, + /// Path to the associated tree state. + state_path: PathBuf, +} + +impl StoredWorkingCopy { + /// Set up a `StoredWorkingCopy`. It's assumed that all paths exist on disk. + fn create( + store: Arc, + output_path: PathBuf, + working_copy_path: PathBuf, + state_path: PathBuf, + ) -> Self { + // Load the tree for our commit. + let state = TreeState::load(store, working_copy_path, state_path).unwrap(); + Self { + state, + output_path, + working_copy_path, + state_path, + } + } +} + +/// The default [`WorkingCopyStore`] for both the Git and native backend. +#[derive(Debug, Default)] +pub struct DefaultWorkingCopyStore { + /// Where the working copies are stored, in this case `.jj/run/default/` + stored_paths: PathBuf, + /// All managed working copies + stored_working_copies: Vec, +} + +/// Creates the required directories for a StoredWorkingCopy. +/// Returns a tuple of (`output_dir`, `working_copy` and `state`). +fn create_working_copy_paths(path: PathBuf) -> Result<(PathBuf, PathBuf, PathBuf), std::io::Error> { + let output = path.join("output"); + let working_copy = path.join("working_copy"); + let state = path.join("state"); + std::fs::create_dir(output)?; + std::fs::create_dir(working_copy)?; + std::fs::create_dir(state)?; + Ok((output, working_copy, state)) +} + +impl DefaultWorkingCopyStore { + fn name() -> &'static str { + "default" + } + + fn init(dot_dir: &Path) -> Self { + let stored_paths = dot_dir.join("run"); + // If the toplevel dir doesn't exist, create it. + if !stored_paths.exists() { + std::fs::create_dir(stored_paths).expect("shouldn't fail"); + } + + Self { + stored_paths, + ..Default::default() + } + } + + fn create_working_copies( + &mut self, + revisions: Vec, + ) -> Result>, std::io::Error> { + let store = revisions + .first() + .expect("revisions shouldn't be empty") + .store(); + // Use the tree id for a unique directory. + for rev in revisions { + let tree_id = rev.tree_id().to_wc_name(); + let path: PathBuf = self.stored_paths.join(tree_id); + // Create a dir under `.jj/run/`. + std::fs::create_dir(path)?; + // And the additional directories. + let (output, working_copy_path, state) = create_working_copy_paths(path)?; + let cached_wc = + StoredWorkingCopy::create(store.clone(), output, working_copy_path, state); + self.stored_working_copies.push(cached_wc); + } + Ok(self.stored_working_copies.clone()) + } +} + +impl WorkingCopyStore for DefaultWorkingCopyStore { + fn as_any(&self) -> dyn std::any::Any { + Box::new(&self) + } + + fn name(&self) -> &'static str { + Self::name() + } + + fn get_or_create_working_copies( + &mut self, + revisions: Vec, + ) -> Vec> { + let new_ids = revisions + .into_iter() + .map(|rev| rev.tree_id().to_wc_name()) + .collect_vec(); + + // check if we're the initial invocation. + let needs_new = if !self.stored_working_copies.is_empty() { + let mut res; + for wc in &self.stored_working_copies { + if !new_ids.contains(&wc.working_copy_path.to_str().unwrap().to_owned()) { + res &= true; + } + } + false + } else { + true + }; + + let result = if !needs_new { + self.stored_working_copies.to_vec() + } else { + self.create_working_copies(revisions).ok().unwrap() + }; + + result + } + + fn has_stores(&self) -> bool { + !self.stored_working_copies.is_empty() + } +} + +impl CachedWorkingCopy for StoredWorkingCopy { + fn exists(&self) -> bool { + self.working_copy_path.exists() && self.state_path.exists() + } + + fn output_path(&self) -> PathBuf { + self.output_path + } +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index cf649466e5..37ee8217ae 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -29,6 +29,7 @@ pub mod default_index_store; pub mod default_revset_engine; pub mod default_revset_graph_iterator; pub mod default_submodule_store; +pub mod default_working_copy_store; pub mod diff; pub mod file_util; pub mod files; @@ -70,4 +71,5 @@ pub mod tree; pub mod tree_builder; pub mod view; pub mod working_copy; +pub mod working_copy_store; pub mod workspace; diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 76810dc2f0..7e1a9c3930 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -60,6 +60,8 @@ use crate::submodule_store::SubmoduleStore; use crate::transaction::Transaction; use crate::tree::TreeMergeError; use crate::view::View; +use crate::working_copy::WorkingCopy; +use crate::working_copy_store::WorkingCopyStore; use crate::{backend, dag_walk, op_store}; pub trait Repo { @@ -73,6 +75,8 @@ pub trait Repo { fn submodule_store(&self) -> &Arc; + fn working_copy_store(&self) -> &Arc; + fn resolve_change_id(&self, change_id: &ChangeId) -> Option> { // Replace this if we added more efficient lookup method. let prefix = HexPrefix::from_bytes(change_id.as_bytes()); @@ -97,6 +101,7 @@ pub struct ReadonlyRepo { settings: RepoSettings, index_store: Arc, submodule_store: Arc, + working_copy_store: Arc, index: OnceCell>>, // Declared after `change_id_index` since it must outlive it on drop. change_id_index: OnceCell>, @@ -141,6 +146,11 @@ impl ReadonlyRepo { &|_settings, store_path| Box::new(DefaultSubmoduleStore::init(store_path)) } + pub fn default_working_copy_store_initializer() -> &'static WorkingCopyStoreInitializer<'static> + { + &|_settings, store_path| Box::new(DefaultWorkingCopyStore::init(store_path)) + } + #[allow(clippy::too_many_arguments)] pub fn init( user_settings: &UserSettings, @@ -151,6 +161,7 @@ impl ReadonlyRepo { op_heads_store_initializer: &OpHeadsStoreInitializer, index_store_initializer: &IndexStoreInitializer, submodule_store_initializer: &SubmoduleStoreInitializer, + working_copy_store_initializer: &WorkingCopyStoreInitializer, ) -> Result, RepoInitError> { let repo_path = repo_path.canonicalize().context(repo_path)?; @@ -208,7 +219,15 @@ impl ReadonlyRepo { .context(&submodule_store_type_path)?; let submodule_store = Arc::from(submodule_store); + let wc_store_path = repo_path.join("working_copy_store"); + fs::create_dir(&wc_store_path).context(&wc_store_path)?; + let wc_store = working_copy_store_initializer(user_settings, &wc_store_path); + let wc_store_type_path = wc_store_path.join("type"); + fs::write(&wc_store_type_path, wc_store.name()).context(&wc_store_type_path)?; + let working_copy_store = Arc::from(wc_store); + let view = View::new(root_view); + Ok(Arc::new(ReadonlyRepo { repo_path, store, @@ -221,6 +240,7 @@ impl ReadonlyRepo { change_id_index: OnceCell::new(), view, submodule_store, + working_copy_store, })) } @@ -233,6 +253,7 @@ impl ReadonlyRepo { op_heads_store: self.op_heads_store.clone(), index_store: self.index_store.clone(), submodule_store: self.submodule_store.clone(), + working_copy_store: self.working_copy_store.clone(), } } @@ -334,6 +355,10 @@ impl Repo for ReadonlyRepo { &self.submodule_store } + fn working_copy_store(&self) -> &Arc { + &self.working_copy_store + } + fn resolve_change_id_prefix(&self, prefix: &HexPrefix) -> PrefixResolution> { self.change_id_index().resolve_prefix(prefix) } @@ -350,6 +375,7 @@ pub type OpHeadsStoreInitializer<'a> = dyn Fn(&UserSettings, &Path) -> Box = dyn Fn(&UserSettings, &Path) -> Box + 'a; pub type SubmoduleStoreInitializer<'a> = dyn Fn(&UserSettings, &Path) -> Box + 'a; +pub type WorkingCopyStoreInitializer<'a> = dyn Fn(&Path) -> Box + 'a; type BackendFactory = Box Result, BackendLoadError>>; @@ -357,6 +383,7 @@ type OpStoreFactory = Box Box>; type OpHeadsStoreFactory = Box Box>; type IndexStoreFactory = Box Box>; type SubmoduleStoreFactory = Box Box>; +type WorkingCopyStoreFactory = Box Box>; pub struct StoreFactories { backend_factories: HashMap, @@ -364,6 +391,7 @@ pub struct StoreFactories { op_heads_store_factories: HashMap, index_store_factories: HashMap, submodule_store_factories: HashMap, + cached_working_copy_factories: HashMap, } impl Default for StoreFactories { @@ -404,6 +432,12 @@ impl Default for StoreFactories { Box::new(|_settings, store_path| Box::new(DefaultSubmoduleStore::load(store_path))), ); + // WorkingCopyStores + factories.add_working_copy_store( + DefaultWorkingCopyStore::name(), + Box::new(|_settings, store_path| Box::new(DefaultWorkingCopyStore::load(store_path))), + ); + factories } } @@ -434,6 +468,7 @@ impl StoreFactories { op_heads_store_factories: HashMap::new(), index_store_factories: HashMap::new(), submodule_store_factories: HashMap::new(), + cached_working_copy_factories: HashMap::new(), } } @@ -564,6 +599,33 @@ impl StoreFactories { Ok(submodule_store_factory(settings, store_path)) } + pub fn add_working_copy_store(&mut self, name: &str, factory: WorkingCopyStoreFactory) { + self.working_copy_store_factories + .insert(name.to_string(), factory); + } + + pub fn load_working_copy_store( + &self, + settings: &UserSettings, + store_path: &Path, + ) -> Result, StoreLoadError> { + // For compatibility with repos without repo/submodule_store. + // TODO Delete default in TBD version + let working_copy_store_type = read_store_type_compat( + "working_copy_store", + store_path.join("type"), + DefaultWorkingCopyStore::name, + )?; + let working_copy_store_factory = self + .cached_working_copy_factories + .get(&working_copy_store_type) + .ok_or_else(|| StoreLoadError::UnsupportedType { + store: "working_copy_store", + store_type: working_copy_store_type.to_string(), + })?; + + Ok(working_copy_store_factory(settings, store_path)) + } } pub fn read_store_type_compat( @@ -604,6 +666,7 @@ pub struct RepoLoader { op_heads_store: Arc, index_store: Arc, submodule_store: Arc, + working_copy_store: Arc, } impl RepoLoader { @@ -629,6 +692,11 @@ impl RepoLoader { store_factories .load_submodule_store(user_settings, &repo_path.join("submodule_store"))?, ); + let working_copy_store = Arc::from( + store_factories + .load_working_copy_store(user_settings, &repo_path.join("submodule_store"))?, + ); + Ok(Self { repo_path: repo_path.to_path_buf(), repo_settings, @@ -637,6 +705,7 @@ impl RepoLoader { op_heads_store, index_store, submodule_store, + working_copy_store, }) } @@ -694,6 +763,7 @@ impl RepoLoader { settings: self.repo_settings.clone(), index_store: self.index_store.clone(), submodule_store: self.submodule_store.clone(), + working_copy_store: self.working_copy_store.clone(), index: OnceCell::with_value(Box::into_pin(index)), change_id_index: OnceCell::new(), view, @@ -726,6 +796,7 @@ impl RepoLoader { settings: self.repo_settings.clone(), index_store: self.index_store.clone(), submodule_store: self.submodule_store.clone(), + working_copy_store: self.working_copy_store.clone(), index: OnceCell::new(), change_id_index: OnceCell::new(), view, diff --git a/lib/src/working_copy_store.rs b/lib/src/working_copy_store.rs new file mode 100644 index 0000000000..ba9f1e8e9a --- /dev/null +++ b/lib/src/working_copy_store.rs @@ -0,0 +1,43 @@ +//! This file stores the [`WorkingCopyStore`] interface and it's associated +//! [`CachedWorkingCopy`] trait. +//! +//! These must be implemented for Virtual Filesystems such as [EdenFS] +//! to allow cheaper working copy materializations, these traits are used for the `jj run` +//! implementation. +//! +//! +//! [EdenFS]: www.github.com/facebook/sapling/main/blob/eden/fs + +use std::{any::Any, path::PathBuf}; + +use crate::commit::Commit; + +/// A `CachedWorkingCopy` is a working copy which is managed by the `WorkingCopyStore`. +pub trait CachedWorkingCopy: Send + Sync { + /// Does the working copy exist. + fn exists(&self) -> bool; + + /// The output path for the this `WorkingCopy`. + /// May look something like `.jj/run/default/{id}/output` + fn output_path(&self) -> PathBuf; +} + +/// A `WorkingCopyStore` manages the working copies on disk for `jj run`. +/// It's an ideal extension point for an virtual filesystem, as they ease the creation of +/// working copies. +pub trait WorkingCopyStore: Send + Sync { + /// Return `self` as `Any` to allow trait upcasting. + fn as_any(&self) -> &dyn Any; + + /// The name of the backend, determines how it actually interacts with files. + fn name(&self) -> &'static str; + + /// Get existing or create `Stores` for `revisions`. + fn get_or_create_working_copies<'a>( + &mut self, + revisions: Vec, + ) -> Vec>; + + /// Are any `Stores` available. + fn has_stores(&self) -> bool; +}