Add support for snapshot/rollback in local machines

src/cartesi-machine.lua

@@ -2106,10 +2106,7 @@ if gdb_address then
 end
 if config.processor.iunrep ~= 0 then stderr("Running in unreproducible mode!\n") end
 if store_config == stderr then store_machine_config(config, stderr) end
-if cmio_advance or cmio_inspect then
-    check_cmio_htif_config(config.htif)
-    assert(remote_address or not perform_rollbacks, "cmio requires --remote-address for snapshot/commit/rollback")
-end
+if cmio_advance or cmio_inspect then check_cmio_htif_config(config.htif) end
 if initial_hash then
     assert(config.processor.iunrep == 0, "hashes are meaningless in unreproducible mode")
     print_root_hash(machine, stderr_unsilenceable)

src/machine-state.h

@@ -44,16 +44,16 @@ struct unpacked_iflags {
 /// \brief Machine state.
 /// \details The machine_state structure contains the entire
 /// state of a Cartesi machine.
-struct machine_state {
-
-    ~machine_state() {
-        ;
-    } // Due to bug in clang++
-
-    /// \brief No copy or move constructor or assignment
-    machine_state(const machine_state &other) = delete;
+struct machine_state final {
+    /// \brief Destructor
+    ~machine_state() = default;
+    /// \brief Copy constructor
+    machine_state(const machine_state &other) = default;
+    /// \brief No move constructor
     machine_state(machine_state &&other) = delete;
+    /// \brief No copy assignment
     machine_state &operator=(const machine_state &other) = delete;
+    /// \brief No move assignment
     machine_state &operator=(machine_state &&other) = delete;
 
     // The following state fields are very hot,

src/machine.cpp

@@ -191,6 +191,11 @@ static bool DID_is_protected(PMA_ISTART_DID DID) {
     }
 }
 
+static uint64_t get_task_concurrency(uint64_t value) {
+    const uint64_t concurrency = value > 0 ? value : std::max(os_get_concurrency(), UINT64_C(1));
+    return std::min(concurrency, static_cast<uint64_t>(THREADS_MAX));
+}
+
 void machine::replace_memory_range(const memory_range_config &range) {
     for (auto &pma : m_s.pmas) {
         if (pma.get_start() == range.start && pma.get_length() == range.length) {
@@ -250,6 +255,49 @@ static void init_tlb_entry(machine &m, uint64_t eidx) {
     tlbce.pma_index = TLB_INVALID_PMA;
 }
 
+template <TLB_entry_type ETYPE>
+static void adjust_tlb_entry(machine &m, uint64_t eidx) {
+    tlb_hot_entry &tlbhe = m.get_state().tlb.hot[ETYPE][eidx];
+    const tlb_cold_entry &tlbce = m.get_state().tlb.cold[ETYPE][eidx];
+    auto vaddr_page = tlbhe.vaddr_page;
+    auto paddr_page = tlbce.paddr_page;
+    auto pma_index = tlbce.pma_index;
+    if (tlbhe.vaddr_page != TLB_INVALID_PAGE) {
+        if ((vaddr_page & ~PAGE_OFFSET_MASK) != vaddr_page) {
+            throw std::invalid_argument{"misaligned virtual page address in TLB entry"};
+        }
+        if ((paddr_page & ~PAGE_OFFSET_MASK) != paddr_page) {
+            throw std::invalid_argument{"misaligned physical page address in TLB entry"};
+        }
+        const pma_entry &pma = m.find_pma_entry<uint64_t>(paddr_page);
+        // Checks if the PMA still valid
+        if (pma.get_length() == 0 || !pma.get_istart_M() || pma_index >= m.get_state().pmas.size() ||
+            &pma != &m.get_state().pmas[pma_index]) {
+            throw std::invalid_argument{"invalid PMA for TLB entry"};
+        }
+        const unsigned char *hpage = pma.get_memory().get_host_memory() + (paddr_page - pma.get_start());
+        // Valid TLB entry
+        tlbhe.vh_offset = cast_ptr_to_addr<uint64_t>(hpage) - vaddr_page;
+    } else {
+        tlbhe.vh_offset = 0;
+    }
+}
+
+void machine::build_pma_list() {
+    // Initialize the vector of the pmas used by the merkle tree to compute hashes.
+    // First, add the pmas visible to the big machine, except the sentinel
+    for (auto &pma : m_s.pmas | sliced(0, m_s.pmas.size() - 1)) {
+        m_pmas.push_back(&pma);
+    }
+
+    // Second, push uarch pmas that are visible only to the microarchitecture interpreter
+    m_pmas.push_back(&m_uarch.get_state().shadow_state);
+    m_pmas.push_back(&m_uarch.get_state().ram);
+
+    // Last, add sentinel
+    m_pmas.push_back(&m_s.empty_pma);
+}
+
 machine::machine(const machine_config &c, const machine_runtime_config &r) :
     m_s{},
     m_t{},
@@ -462,18 +510,8 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) :
     // Add sentinel to PMA vector
     register_pma_entry(make_empty_pma_entry("sentinel"s, 0, 0));
 
-    // Initialize the vector of the pmas used by the merkle tree to compute hashes.
-    // First, add the pmas visible to the big machine, except the sentinel
-    for (auto &pma : m_s.pmas | sliced(0, m_s.pmas.size() - 1)) {
-        m_pmas.push_back(&pma);
-    }
-
-    // Second, push uarch pmas that are visible only to the microarchitecture interpreter
-    m_pmas.push_back(&m_uarch.get_state().shadow_state);
-    m_pmas.push_back(&m_uarch.get_state().ram);
-
-    // Last, add sentinel
-    m_pmas.push_back(&m_s.empty_pma);
+    // Populate m_pmas
+    build_pma_list();
 
     // Initialize TLB device
     // this must be done after all PMA entries are already registered, so we can lookup page addresses
@@ -543,6 +581,83 @@ machine::machine(const std::string &dir, const machine_runtime_config &r) : mach
     }
 }
 
+machine::machine(const machine &other) :
+    m_s(other.m_s),
+    m_t(),
+    m_pmas(),
+    m_c(other.m_c),
+    m_uarch(other.m_uarch),
+    m_r(other.m_r),
+    m_mrds(other.m_mrds) {
+
+    // Cannot copy machine with VirtIO devices
+    if (!other.m_vdevs.empty()) {
+        throw std::runtime_error{"cannot copy machine with VirtIO devices"};
+    }
+
+    // Populate m_pmas
+    build_pma_list();
+
+    // ??(edubart) Mark all pages as dirty for now, we need to implement a deep copy for merkle tree later.
+    for (auto *pma : m_pmas) {
+        pma->mark_pages_dirty();
+    }
+
+    // Clone memory mapped PMAs
+    for (auto *pma : m_pmas) {
+        // Need to remap only memory
+        if (pma->get_istart_M()) {
+            auto &mem = pma->get_memory();
+
+            // Map a new PMA memory with the same size
+            pma_memory cloned_mem(pma->get_description(), mem.get_length(), pma_memory::callocd{});
+            const unsigned char *data = mem.get_host_memory();
+            unsigned char *cloned_data = cloned_mem.get_host_memory();
+
+            // Copy memory from the old PMA to the new PMA memory
+            const uint64_t num_threads = get_task_concurrency(m_r.concurrency.update_merkle_tree);
+            if (num_threads == 1) {
+                memcpy(cloned_data, data, mem.get_length());
+            } else { // Use multiple threads to copy the memory
+                const uint64_t pages_in_range = (pma->get_length() + PMA_PAGE_SIZE - 1) / PMA_PAGE_SIZE;
+                const uint64_t pages_per_thread = pages_in_range / num_threads;
+                if (pages_per_thread == 0) {
+                    memcpy(cloned_mem.get_host_memory(), mem.get_host_memory(), mem.get_length());
+                } else {
+                    const uint64_t rest_pages = pages_per_thread % num_threads;
+                    const uint64_t chunk_len = pages_per_thread * PMA_PAGE_SIZE;
+                    // Let each thread copy a chunk of memory
+                    const bool succeeded =
+                        os_parallel_for(num_threads, [&](int i, const parallel_for_mutex &mutex) -> bool {
+                            (void) mutex;
+                            const uint64_t page_start = i * chunk_len;
+                            memcpy(cloned_data + page_start, data + page_start, chunk_len);
+                            return true;
+                        });
+                    if (!succeeded) {
+                        throw std::runtime_error{"copy memory parallel for failed"};
+                    }
+                    // Copy remaining pages
+                    if (rest_pages > 0) {
+                        const uint64_t page_start = num_threads * chunk_len;
+                        memcpy(cloned_data, data, mem.get_length() - page_start);
+                    }
+                }
+            }
+
+            // Replace the PMA with the new mapped PMA
+            mem.replace(std::move(cloned_mem));
+        }
+    }
+
+    // Adjust TLB offsets to host memory pointers
+    for (uint64_t i = 0; i < PMA_TLB_SIZE; ++i) {
+        adjust_tlb_entry<TLB_CODE>(*this, i);
+        adjust_tlb_entry<TLB_READ>(*this, i);
+        adjust_tlb_entry<TLB_WRITE>(*this, i);
+    }
+}
+
 void machine::prepare_virtio_devices_select(select_fd_sets *fds, uint64_t *timeout_us) {
     for (auto &vdev : m_vdevs) {
         vdev->prepare_select(fds, timeout_us);
@@ -1550,11 +1665,6 @@ bool machine::verify_dirty_page_maps(void) const {
     return !broken;
 }
 
-static uint64_t get_task_concurrency(uint64_t value) {
-    const uint64_t concurrency = value > 0 ? value : std::max(os_get_concurrency(), UINT64_C(1));
-    return std::min(concurrency, static_cast<uint64_t>(THREADS_MAX));
-}
-
 bool machine::update_merkle_tree(void) const {
     machine_merkle_tree::hasher_type gh;
     static_assert(PMA_PAGE_SIZE == machine_merkle_tree::get_page_size(),

src/machine.h

@@ -72,6 +72,9 @@ class machine final {
     static const pma_entry::flags m_cmio_rx_buffer_flags; ///< PMA flags used for cmio rx buffer
     static const pma_entry::flags m_cmio_tx_buffer_flags; ///< PMA flags used for cmio tx buffer
 
+    /// \brief Build PMA list from machine state and uarch machine state.
+    void build_pma_list();
+
     /// \brief Allocates a new PMA entry.
     /// \param pma PMA entry to add to machine.
     /// \returns Reference to corresponding entry in machine state.
@@ -186,8 +189,8 @@ class machine final {
 
     /// \brief No default constructor
     machine(void) = delete;
-    /// \brief No copy constructor
-    machine(const machine &other) = delete;
+    /// \brief Copy constructor
+    machine(const machine &other);
     /// \brief No move constructor
     machine(machine &&other) = delete;
     /// \brief No copy assignment

src/os.cpp

@@ -524,7 +524,29 @@ int os_mkdir(const char *path, int mode) {
 #endif // HAVE_MKDIR
 }
 
-unsigned char *os_map_file(const char *path, uint64_t length, bool shared) {
+mmapd os_mmap_mem(uint64_t length) {
+    if (length == 0) {
+        return {nullptr, 0, -1, false};
+    }
+
+#ifdef HAVE_MMAP
+    auto *host_memory =
+        static_cast<unsigned char *>(mmap(nullptr, length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
+    if (host_memory == MAP_FAILED) { // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,performance-no-int-to-ptr)
+        throw std::system_error{errno, std::generic_category(), "could not map memory file"s};
+    }
+    return {host_memory, length, -1, false};
+
+#else
+    // Use calloc to improve performance and to initialize it to zeros.
+    // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, cppcoreguidelines-prefer-member-initializer)
+    auto *host_memory = static_cast<unsigned char *>(std::calloc(1, length));
+    return {host_memory, length, -1, false};
+
+#endif // HAVE_MMAP
+}
+
+mmapd os_mmap_file(const char *path, uint64_t length, bool shared) {
     if (!path || *path == '\0') {
         throw std::runtime_error{"image file path must be specified"s};
     }
@@ -563,9 +585,7 @@ unsigned char *os_map_file(const char *path, uint64_t length, bool shared) {
         throw std::system_error{errno, std::generic_category(), "could not map image file '"s + path + "' to memory"s};
     }
 
-    // We can close the file after mapping it, because the OS will retain a reference of the file on its own
-    close(backing_file);
-    return host_memory;
+    return {host_memory, length, backing_file, shared};
 
 #elif defined(_WIN32)
     const int oflag = (shared ? _O_RDWR : _O_RDONLY) | _O_BINARY;
@@ -609,9 +629,7 @@ unsigned char *os_map_file(const char *path, uint64_t length, bool shared) {
         throw std::system_error{errno, std::generic_category(), "could not map image file '"s + path + "' to memory"s};
     }
 
-    // We can close the file after mapping it, because the OS will retain a reference of the file on its own
-    _close(backing_file);
-    return host_memory;
+    return {host_memory, length, backing_file, shared};
 
 #else
     if (shared) {
@@ -637,7 +655,7 @@ unsigned char *os_map_file(const char *path, uint64_t length, bool shared) {
         throw std::runtime_error{"image file '"s + path + "' of "s + " is too large for range"s};
     }
 
-    // use calloc to improve performance
+    // Use calloc to improve performance and to initialize it to zeros.
     // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, cppcoreguidelines-prefer-member-initializer)
     auto host_memory = static_cast<unsigned char *>(std::calloc(1, length));
     if (!host_memory) {
@@ -650,22 +668,27 @@ unsigned char *os_map_file(const char *path, uint64_t length, bool shared) {
     if (ferror(fp.get())) {
         throw std::system_error{errno, std::generic_category(), "error reading from image file '"s + path + "'"s};
     }
-    return host_memory;
+    return {host_memory, length, -1, false};
 
 #endif // HAVE_MMAP
 }
 
-void os_unmap_file(unsigned char *host_memory, uint64_t length) {
+void os_munmap(const mmapd &mem) {
 #ifdef HAVE_MMAP
-    munmap(host_memory, length);
+    munmap(mem.host_memory, mem.length);
+    if (mem.backing_file != -1) {
+        close(mem.backing_file);
+    }
 
 #elif defined(_WIN32)
     (void) length;
-    UnmapViewOfFile(host_memory);
+    UnmapViewOfFile(mem.host_memory);
+    if (mem.backing_file != -1) {
+        _close(mem.backing_file);
+    }
 
 #else
-    (void) length;
-    std::free(host_memory);
+    std::free(mem.host_memory);
 
 #endif // HAVE_MMAP
 }

src/os.h

@@ -88,11 +88,22 @@ void os_putchars(const uint8_t *data, size_t len);
 /// \brief Creates a new directory
 int os_mkdir(const char *path, int mode);
 
-/// \brief Maps a file to memory
-unsigned char *os_map_file(const char *path, uint64_t length, bool shared);
+/// \brief Mapped memory entry
+struct mmapd {
+    unsigned char *host_memory = nullptr; ///< Start of associated memory region in host.
+    uint64_t length = 0;                  ///< Length of memory range (copy of PMA length field).
+    int backing_file = -1;                ///< File descriptor backing the memory
+    bool shared = false;                  ///< True when the memory is shared with the file descriptor
+};
+
+/// \brief Maps memory
+mmapd os_mmap_mem(uint64_t length);
+
+/// \brief Maps memory from a file
+mmapd os_mmap_file(const char *path, uint64_t length, bool shared);
 
-/// \brief Unmaps a file from memory
-void os_unmap_file(unsigned char *host_memory, uint64_t length);
+/// \brief Unmaps memory
+void os_munmap(const mmapd &m);
 
 /// \brief Get time elapsed since its first call with microsecond precision
 int64_t os_now_us();