feat(sozo): support for dev mode with hot reloading (#890)

* feat(sozo): #557: suport dev mode with hot reloading.

Build is triggered for any modification to Scarb.toml or a filename with 'cairo' extension

* sozo: add DojoCompiler for build and dev subcommand

* sozo: add copy of non pub functions from scarb required for hot loading

* sozo: rebuild project when a cairo file is modified

* sozo: add debouncer for dev mode

* feat(sozo): #557: update Cargo.lock

* feat(sozo): #557: support dev mode with hot reloading

Add migration

* make it compile

* fix formatting

* remove TODO

* fix file copied from scarb

* fix formatting

* update manifest on migrate

* add NOTE on scarb_internal module

---------

Co-authored-by: Patrice Tisserand <[email protected]>

crates/sozo/Cargo.toml

@@ -9,6 +9,7 @@ version = "0.2.1"
 anyhow.workspace = true
 async-trait.workspace = true
 cairo-lang-compiler.workspace = true
+cairo-lang-defs.workspace = true
 cairo-lang-filesystem.workspace = true
 cairo-lang-plugins.workspace = true
 cairo-lang-project.workspace = true
@@ -24,6 +25,8 @@ console.workspace = true
 dojo-lang = { path = "../dojo-lang" }
 dojo-types = { path = "../dojo-types" }
 dojo-world = { path = "../dojo-world" }
+notify = "6.0.1"
+notify-debouncer-mini = "0.3.0"
 scarb-ui.workspace = true
 scarb.workspace = true
 semver.workspace = true

crates/sozo/src/args.rs

@@ -11,6 +11,7 @@ use crate::commands::auth::AuthArgs;
 use crate::commands::build::BuildArgs;
 use crate::commands::completions::CompletionsArgs;
 use crate::commands::component::ComponentArgs;
+use crate::commands::dev::DevArgs;
 use crate::commands::events::EventsArgs;
 use crate::commands::execute::ExecuteArgs;
 use crate::commands::init::InitArgs;
@@ -57,6 +58,8 @@ pub enum Commands {
     #[command(about = "Run a migration, declaring and deploying contracts as necessary to \
                        update the world")]
     Migrate(Box<MigrateArgs>),
+    #[command(about = "Developer mode: watcher for building and migration")]
+    Dev(DevArgs),
     #[command(about = "Test the project's smart contracts")]
     Test(TestArgs),
     #[command(about = "Execute a world's system")]

crates/sozo/src/commands/dev.rs

@@ -0,0 +1,273 @@
+use std::mem;
+use std::path::{Path, PathBuf};
+use std::sync::mpsc::channel;
+use std::time::Duration;
+
+use anyhow::{anyhow, Result};
+use cairo_lang_compiler::db::RootDatabase;
+use cairo_lang_filesystem::db::{AsFilesGroupMut, FilesGroupEx, PrivRawFileContentQuery};
+use cairo_lang_filesystem::ids::FileId;
+use clap::Args;
+use dojo_world::manifest::Manifest;
+use dojo_world::metadata::{dojo_metadata_from_workspace, DojoMetadata};
+use dojo_world::migration::world::WorldDiff;
+use notify_debouncer_mini::notify::RecursiveMode;
+use notify_debouncer_mini::{new_debouncer, DebouncedEvent, DebouncedEventKind};
+use scarb::compiler::CompilationUnit;
+use scarb::core::{Config, Workspace};
+use starknet::accounts::SingleOwnerAccount;
+use starknet::core::types::FieldElement;
+use starknet::providers::Provider;
+use starknet::signers::Signer;
+use tracing_log::log;
+
+use super::options::account::AccountOptions;
+use super::options::starknet::StarknetOptions;
+use super::options::world::WorldOptions;
+use super::scarb_internal::build_scarb_root_database;
+use crate::ops::migration;
+
+#[derive(Args)]
+pub struct DevArgs {
+    #[arg(long)]
+    #[arg(help = "Name of the World.")]
+    #[arg(long_help = "Name of the World. It's hash will be used as a salt when deploying the \
+                       contract to avoid address conflicts.")]
+    pub name: Option<String>,
+
+    #[command(flatten)]
+    pub world: WorldOptions,
+
+    #[command(flatten)]
+    pub starknet: StarknetOptions,
+
+    #[command(flatten)]
+    pub account: AccountOptions,
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
+enum DevAction {
+    None,
+    Reload,
+    Build(PathBuf),
+}
+
+fn handle_event(event: &DebouncedEvent) -> DevAction {
+    let action = match event.kind {
+        DebouncedEventKind::Any => {
+            let p = event.path.clone();
+            if let Some(filename) = p.file_name() {
+                if filename == "Scarb.toml" {
+                    return DevAction::Reload;
+                } else if let Some(extension) = p.extension() {
+                    if extension == "cairo" {
+                        return DevAction::Build(p.clone());
+                    }
+                }
+            }
+            DevAction::None
+        }
+        _ => DevAction::None,
+    };
+    action
+}
+
+struct DevContext<'a> {
+    pub db: RootDatabase,
+    pub unit: CompilationUnit,
+    pub ws: Workspace<'a>,
+    pub dojo_metadata: Option<DojoMetadata>,
+}
+
+fn load_context(config: &Config) -> Result<DevContext<'_>> {
+    let ws = scarb::ops::read_workspace(config.manifest_path(), config)?;
+    let packages: Vec<scarb::core::PackageId> = ws.members().map(|p| p.id).collect();
+    let resolve = scarb::ops::resolve_workspace(&ws)?;
+    let compilation_units = scarb::ops::generate_compilation_units(&resolve, &ws)?
+        .into_iter()
+        .filter(|cu| packages.contains(&cu.main_package_id))
+        .collect::<Vec<_>>();
+    // we have only 1 unit in projects
+    let unit = compilation_units.get(0).unwrap();
+    let db = build_scarb_root_database(unit, &ws).unwrap();
+    let dojo_metadata = dojo_metadata_from_workspace(&ws);
+    Ok(DevContext { db, unit: unit.clone(), ws, dojo_metadata })
+}
+
+fn build(context: &mut DevContext<'_>) -> Result<()> {
+    let ws = &context.ws;
+    let unit = &context.unit;
+    let package_name = unit.main_package_id.name.clone();
+    ws.config().compilers().compile(unit.clone(), &mut (context.db), ws).map_err(|err| {
+        ws.config().ui().anyhow(&err);
+
+        anyhow!("could not compile `{package_name}` due to previous error")
+    })?;
+    ws.config().ui().print("📦 Rebuild done");
+    Ok(())
+}
+
+async fn migrate<P, S>(
+    mut world_address: Option<FieldElement>,
+    account: &SingleOwnerAccount<P, S>,
+    name: Option<String>,
+    ws: &Workspace<'_>,
+    previous_manifest: Option<Manifest>,
+) -> Result<(Manifest, Option<FieldElement>)>
+where
+    P: Provider + Sync + Send + 'static,
+    S: Signer + Sync + Send + 'static,
+{
+    let target_dir = ws.target_dir().path_existent().unwrap();
+    let target_dir = target_dir.join(ws.config().profile().as_str());
+    let manifest_path = target_dir.join("manifest.json");
+    if !manifest_path.exists() {
+        return Err(anyhow!("manifest.json not found"));
+    }
+    let new_manifest = Manifest::load_from_path(manifest_path)?;
+    let diff = WorldDiff::compute(new_manifest.clone(), previous_manifest);
+    let total_diffs = diff.count_diffs();
+    let config = ws.config();
+    config.ui().print(format!("Total diffs found: {total_diffs}"));
+    if total_diffs == 0 {
+        return Ok((new_manifest, world_address));
+    }
+    match migration::apply_diff(
+        target_dir,
+        diff,
+        name.clone(),
+        world_address,
+        account,
+        config,
+        None,
+    )
+    .await
+    {
+        Ok(address) => {
+            config
+                .ui()
+                .print(format!("🎉 World at address {} updated!", format_args!("{:#x}", address)));
+            world_address = Some(address);
+        }
+        Err(err) => {
+            config.ui().error(err.to_string());
+            return Err(err);
+        }
+    }
+
+    Ok((new_manifest, world_address))
+}
+
+fn process_event(event: &DebouncedEvent, context: &mut DevContext<'_>) -> DevAction {
+    let action = handle_event(event);
+    match &action {
+        DevAction::None => {}
+        DevAction::Build(path) => handle_build_action(path, context),
+        DevAction::Reload => {
+            handle_reload_action(context);
+        }
+    }
+    action
+}
+
+fn handle_build_action(path: &Path, context: &mut DevContext<'_>) {
+    context
+        .ws
+        .config()
+        .ui()
+        .print(format!("📦 Need to rebuild {}", path.to_str().unwrap_or_default(),));
+    let db = &mut context.db;
+    let file = FileId::new(db, path.to_path_buf());
+    PrivRawFileContentQuery.in_db_mut(db.as_files_group_mut()).invalidate(&file);
+    db.override_file_content(file, None);
+}
+
+fn handle_reload_action(context: &mut DevContext<'_>) {
+    let config = context.ws.config();
+    config.ui().print("Reloading project");
+    let new_context = load_context(config).expect("Failed to load context");
+    let _ = mem::replace(context, new_context);
+}
+
+impl DevArgs {
+    pub fn run(self, config: &Config) -> Result<()> {
+        let mut context = load_context(config)?;
+        let (tx, rx) = channel();
+        let mut debouncer = new_debouncer(Duration::from_secs(1), None, tx)?;
+
+        debouncer.watcher().watch(
+            config.manifest_path().parent().unwrap().as_std_path(),
+            RecursiveMode::Recursive,
+        )?;
+        let name = self.name.clone();
+        let mut previous_manifest: Option<Manifest> = Option::None;
+        let result = build(&mut context);
+        let env_metadata = context.dojo_metadata.as_ref().and_then(|e| e.env.clone());
+
+        let Some((mut world_address, account)) = context
+            .ws
+            .config()
+            .tokio_handle()
+            .block_on(migration::setup_env(
+                self.account,
+                self.starknet,
+                self.world,
+                env_metadata.as_ref(),
+                config,
+                name.as_ref(),
+            ))
+            .ok()
+        else {
+            return Err(anyhow!("Failed to setup environment"));
+        };
+
+        match context.ws.config().tokio_handle().block_on(migrate(
+            world_address,
+            &account,
+            name.clone(),
+            &context.ws,
+            previous_manifest.clone(),
+        )) {
+            Ok((manifest, address)) => {
+                previous_manifest = Some(manifest);
+                world_address = address;
+            }
+            Err(error) => {
+                log::error!("Error: {error:?}");
+            }
+        }
+        loop {
+            let action = match rx.recv() {
+                Ok(Ok(events)) => events
+                    .iter()
+                    .map(|event| process_event(event, &mut context))
+                    .last()
+                    .unwrap_or(DevAction::None),
+                Ok(Err(_)) => DevAction::None,
+                Err(error) => {
+                    log::error!("Error: {error:?}");
+                    break;
+                }
+            };
+
+            if action != DevAction::None && build(&mut context).is_ok() {
+                match context.ws.config().tokio_handle().block_on(migrate(
+                    world_address,
+                    &account,
+                    name.clone(),
+                    &context.ws,
+                    previous_manifest.clone(),
+                )) {
+                    Ok((manifest, address)) => {
+                        previous_manifest = Some(manifest);
+                        world_address = address;
+                    }
+                    Err(error) => {
+                        log::error!("Error: {error:?}");
+                    }
+                }
+            }
+        }
+        result
+    }
+}

crates/sozo/src/commands/mod.rs

@@ -7,6 +7,7 @@ pub(crate) mod auth;
 pub(crate) mod build;
 pub(crate) mod completions;
 pub(crate) mod component;
+pub(crate) mod dev;
 pub(crate) mod events;
 pub(crate) mod execute;
 pub(crate) mod init;
@@ -16,13 +17,16 @@ pub(crate) mod register;
 pub(crate) mod system;
 pub(crate) mod test;
 
+// copy of non pub functions from scarb
+pub(crate) mod scarb_internal;
+
 pub fn run(command: Commands, config: &Config) -> Result<()> {
     match command {
         Commands::Init(args) => args.run(config),
         Commands::Test(args) => args.run(config),
         Commands::Build(args) => args.run(config),
         Commands::Migrate(args) => args.run(config),
-
+        Commands::Dev(args) => args.run(config),
         Commands::Auth(args) => args.run(config),
         Commands::Execute(args) => args.run(config),
         Commands::Component(args) => args.run(config),

crates/sozo/src/commands/scarb_internal/mod.rs

@@ -0,0 +1,55 @@
+// I have copied source code from https://github.com/software-mansion/scarb/blob/main/scarb/src/compiler/db.rs
+// since build_scarb_root_database is not public
+//
+// NOTE: This files needs to be updated whenever scarb version is updated
+use anyhow::Result;
+use cairo_lang_compiler::db::RootDatabase;
+use cairo_lang_compiler::project::{ProjectConfig, ProjectConfigContent};
+use cairo_lang_filesystem::ids::Directory;
+use scarb::compiler::CompilationUnit;
+use scarb::core::Workspace;
+use tracing::trace;
+
+// TODO(mkaput): ScarbDatabase?
+pub(crate) fn build_scarb_root_database(
+    unit: &CompilationUnit,
+    ws: &Workspace<'_>,
+) -> Result<RootDatabase> {
+    let mut b = RootDatabase::builder();
+    b.with_project_config(build_project_config(unit)?);
+    b.with_cfg(unit.cfg_set.clone());
+
+    for plugin_info in &unit.cairo_plugins {
+        let package_id = plugin_info.package.id;
+        let plugin = ws.config().cairo_plugins().fetch(package_id)?;
+        let instance = plugin.instantiate()?;
+        for macro_plugin in instance.macro_plugins() {
+            b.with_macro_plugin(macro_plugin);
+        }
+        for (name, inline_macro_plugin) in instance.inline_macro_plugins() {
+            b.with_inline_macro_plugin(&name, inline_macro_plugin);
+        }
+    }
+
+    b.build()
+}
+
+fn build_project_config(unit: &CompilationUnit) -> Result<ProjectConfig> {
+    let crate_roots = unit
+        .components
+        .iter()
+        .filter(|component| !component.package.id.is_core())
+        .map(|component| (component.cairo_package_name(), component.target.source_root().into()))
+        .collect();
+
+    let corelib = Some(Directory::Real(unit.core_package_component().target.source_root().into()));
+
+    let content = ProjectConfigContent { crate_roots };
+
+    let project_config =
+        ProjectConfig { base_path: unit.main_component().package.root().into(), corelib, content };
+
+    trace!(?project_config);
+
+    Ok(project_config)
+}