Why RaftEngine fdatasync so fast?

[Neyzoter]

I wrote a benchmark to test writing data to dis, 100KB data cost 20ms. But RaftEngine syncing data cost <5ms.

So, Why RaftEngine so fast?

my code

// my code
use std::borrow::BorrowMut;
use std::fs::File;
use std::io::{Write, Error, SeekFrom, Seek};
use std::os::macos::fs::MetadataExt;
use std::ptr::addr_of_mut;
use std::{fs, time};
use std::path::Path;

fn write_to_file(data: &[u8], file: &mut File) -> Result<(), Error> {
    file.write_all(data)?;
    file.sync_all()?;
    Ok(())
}

fn main() {
    let file_paths = ["/Users/song/code/personal/rustacean/write_perf/src/10",
        "/Users/song/code/personal/rustacean/write_perf/src/50",
        "/Users/song/code/personal/rustacean/write_perf/src/100",
        "/Users/song/code/personal/rustacean/write_perf/src/200"];
    for file_path in file_paths {
        let path = Path::new(file_path);
        if path.exists() {
            fs::remove_file(file_path).unwrap();
        }
        let mut file = File::create(file_path).unwrap();
        let name = path.file_name().unwrap().to_str().unwrap();
        let kb:i32 = name.parse().unwrap();
        let mut data = vec![];
        for i in 0..(1024*kb) {
            data.push(48)
        }
        println!("\n------");
        println!("{kb}KB:");
        for i in 0..5 {
            file.seek(SeekFrom::Start(0)).unwrap();
            let st = time::SystemTime::now();
            write_to_file(&data, file.borrow_mut()).unwrap();
            let et = time::SystemTime::now();
            let delta = et.duration_since(st).unwrap();
            println!("  |- {i} : {} us", delta.as_micros())
        }
    }
}

output:

------
10KB:
  |- 0 : 22458 us
  |- 1 : 18801 us
  |- 2 : 20117 us
  |- 3 : 22492 us
  |- 4 : 19413 us

------
50KB:
  |- 0 : 18804 us
  |- 1 : 19410 us
  |- 2 : 19550 us
  |- 3 : 20961 us
  |- 4 : 18579 us

------
100KB:
  |- 0 : 20376 us
  |- 1 : 19297 us
  |- 2 : 19636 us
  |- 3 : 18982 us
  |- 4 : 19424 us

------
200KB:
  |- 0 : 31969 us
  |- 1 : 19855 us
  |- 2 : 19587 us
  |- 3 : 19023 us
  |- 4 : 18943 us

raft engine

// write 100KB data to disk, based on RaftEngine example.

use std::time;
use std::time::SystemTime;
use kvproto::raft_serverpb::RaftLocalState;
use raft::eraftpb::Entry;
use raft_engine::{Config, Engine, LogBatch, MessageExt, ReadableSize};
use rand::thread_rng;
use rand_distr::{Distribution, Normal};

#[derive(Clone)]
pub struct MessageExtTyped;

impl MessageExt for MessageExtTyped {
    type Entry = Entry;

    fn index(e: &Self::Entry) -> u64 {
        e.index
    }
}

// How to run the example:
// $ RUST_LOG=debug cargo run --release --example append-compact-purge
fn main() {
    env_logger::init();

    let config = Config {
        dir: "append-compact-purge-data".to_owned(),
        purge_threshold: ReadableSize::gb(2),
        batch_compression_threshold: ReadableSize::kb(0),
        ..Default::default()
    };
    let engine = Engine::open(config).expect("Open raft engine");

    let compact_offset = 32; // In src/purge.rs, it's the limit for rewrite.

    let mut rand_regions = Normal::new(128.0, 96.0)
        .unwrap()
        .sample_iter(thread_rng())
        .map(|x| x as u64);
    let mut rand_compacts = Normal::new(compact_offset as f64, 16.0)
        .unwrap()
        .sample_iter(thread_rng())
        .map(|x| x as u64);

    let mut batch = LogBatch::with_capacity(256);
    let mut entry = Entry::new();
    entry.set_data(vec![b'x'; 1024 * 100].into());
    let init_state = RaftLocalState {
        last_index: 0,
        ..Default::default()
    };

    let region = rand_regions.next().unwrap();
    let mut state = engine
        .get_message::<RaftLocalState>(region, b"last_index")
        .unwrap()
        .unwrap_or_else(|| init_state.clone());

    state.last_index += 1; // manually update the state
    let mut e = entry.clone();
    e.index = state.last_index;
    batch.add_entries::<MessageExtTyped>(region, &[e]).unwrap();
    batch
        .put_message(region, b"last_index".to_vec(), &state)
        .unwrap();
    let st = time::SystemTime::now();
    let batch_size = batch.approximate_size();
    engine.write(&mut batch, true).unwrap();
    let et = time::SystemTime::now();
    let delta = et.duration_since(st).unwrap();
    println!("engine write {} bytes data, cost {} us", batch_size, delta.as_micros());
}

output:

engine write 102509 bytes data, cost 1024 us

[tabokie]

A few reasons I can think of right now:

• We use fallocate to extend file in advance:

raft-engine/src/file_pipe_log/log_file.rs

Lines 96 to 107 in 2c76690

	if self.capacity < new_written {
	let _t = StopWatch::new(&*LOG_ALLOCATE_DURATION_HISTOGRAM);
	let alloc = std::cmp::min(
	FILE_ALLOCATE_SIZE,
	target_size_hint.saturating_sub(self.capacity),
	);
	let alloc = std::cmp::max(new_written - self.capacity, alloc);
	if let Err(e) = self.writer.allocate(self.capacity, alloc) {
	warn!("log file allocation failed: {e}");
	}
	self.capacity += alloc;
	}

• We use fdatasync whereas std::File::sync_all uses fsync.

[Neyzoter]

macos: File::sync_data() call libc::fcntl(fd, libc::F_FULLFSYNC). Latency is big.macos unsupport libc::fdatasync(fd).
centos: File::sync_data() call libc::fdatasync(fd).Latency is small.

use nix::unistd::{fsync};
use std::borrow::{Borrow, BorrowMut};
use std::fs::File;
use std::io::{Write, Error, SeekFrom, Seek};
use std::ptr::addr_of_mut;
use std::{fs, time};
use std::os::unix::io::{AsRawFd, RawFd};
use std::path::Path;
use std::thread::sleep;
use nix::fcntl;

const CIRCLE:i32 = 3;
const SYNC_METHOD: [SyncMethod; 3] = [SyncMethod::LibcSync, SyncMethod::RustSyncData, SyncMethod::RustSync];

#[derive(Debug)]
enum SyncMethod {
    LibcSyncData,  // macos unsupported
    LibcSync,
    RustSyncData,
    RustSync,
    FcntlFullSync
}
fn sync(fd: RawFd) -> std::io::Result<()> {
    unsafe {
        fsync(fd)?;
    };
    Ok(())
}

fn write_to_file(data: &[u8], file: &mut File, sm: &SyncMethod) -> Result<(), Error> {
    file.write(data)?;
    match sm {
        SyncMethod::LibcSync => {
            sync(file.as_raw_fd())?;
        },
        SyncMethod::RustSyncData => {
            file.sync_data()?;
        },
        SyncMethod::RustSync => {
            file.sync_all()?;
        },
        _ => {}
    }
    Ok(())
}


fn main() {
    let file_paths = ["./test/10",
        "./test/50",
        "./test/100",
        "./test/200",
        "./test/500",
        "./test/1000"];
    for sm in SYNC_METHOD {
        println!("\n---{:?}---", sm.borrow());
        for file_path in file_paths {
            let path = Path::new(file_path);
            let name = path.file_name().unwrap().to_str().unwrap();
            let kb:i32 = name.parse().unwrap();
            let mut data = vec![];
            for _ in 0..(1024*kb) {
                data.push(b'x');
            }
            println!("  |- {kb}KB:");
            for i in 0..CIRCLE {
                if path.exists() {
                    fs::remove_file(file_path).unwrap();
                    while path.exists() {
                        sleep(time::Duration::from_millis(100))
                    }
                }
                let mut file = File::create(file_path).unwrap();
                // file.set_len((1024 * kb) as u64).unwrap(); // 预先设置空间
                let st = time::SystemTime::now();
                write_to_file(&data, file.borrow_mut(), &sm).unwrap();
                let et = time::SystemTime::now();
                let delta = et.duration_since(st).unwrap();
                println!("    |- {i} : {} us", delta.as_micros());
            }
        }
    }
}

macos output:

---LibcSync---
  |- 10KB:
    |- 0 : 658 us
    |- 1 : 202 us
    |- 2 : 160 us
  |- 50KB:
    |- 0 : 176 us
    |- 1 : 192 us
    |- 2 : 208 us
  |- 100KB:
    |- 0 : 349 us
    |- 1 : 256 us
    |- 2 : 161 us
  |- 200KB:
    |- 0 : 319 us
    |- 1 : 294 us
    |- 2 : 291 us
  |- 500KB:
    |- 0 : 481 us
    |- 1 : 416 us
    |- 2 : 443 us
  |- 1000KB:
    |- 0 : 759 us
    |- 1 : 725 us
    |- 2 : 781 us

---RustSyncData---
  |- 10KB:
    |- 0 : 19292 us
    |- 1 : 19783 us
    |- 2 : 19469 us
  |- 50KB:
    |- 0 : 26152 us
    |- 1 : 19253 us
    |- 2 : 20261 us
  |- 100KB:
    |- 0 : 19756 us
    |- 1 : 20250 us
    |- 2 : 20128 us
  |- 200KB:
    |- 0 : 36114 us
    |- 1 : 21687 us
    |- 2 : 19395 us
  |- 500KB:
    |- 0 : 21550 us
    |- 1 : 38844 us
    |- 2 : 24180 us
  |- 1000KB:
    |- 0 : 27402 us
    |- 1 : 20342 us
    |- 2 : 40084 us

---RustSync---
  |- 10KB:
    |- 0 : 23191 us
    |- 1 : 40821 us
    |- 2 : 19228 us
  |- 50KB:
    |- 0 : 19843 us
    |- 1 : 19028 us
    |- 2 : 19346 us
  |- 100KB:
    |- 0 : 20373 us
    |- 1 : 20403 us
    |- 2 : 20127 us
  |- 200KB:
    |- 0 : 25280 us
    |- 1 : 20326 us
    |- 2 : 19623 us
  |- 500KB:
    |- 0 : 19747 us
    |- 1 : 19469 us
    |- 2 : 19824 us
  |- 1000KB:
    |- 0 : 20973 us
    |- 1 : 21453 us
    |- 2 : 20563 us

centos output:

---LibcSync---
  |- 10KB:
    |- 0 : 1287 us
    |- 1 : 1246 us
    |- 2 : 1163 us
  |- 50KB:
    |- 0 : 1263 us
    |- 1 : 1319 us
    |- 2 : 1225 us
  |- 100KB:
    |- 0 : 1366 us
    |- 1 : 1353 us
    |- 2 : 1359 us
  |- 200KB:
    |- 0 : 1580 us
    |- 1 : 1474 us
    |- 2 : 1554 us
  |- 500KB:
    |- 0 : 2292 us
    |- 1 : 2368 us
    |- 2 : 2140 us
  |- 1000KB:
    |- 0 : 3593 us
    |- 1 : 3450 us
    |- 2 : 3302 us

---RustSyncData---
  |- 10KB:
    |- 0 : 1307 us
    |- 1 : 1150 us
    |- 2 : 1165 us
  |- 50KB:
    |- 0 : 1267 us
    |- 1 : 1257 us
    |- 2 : 1225 us
  |- 100KB:
    |- 0 : 1657 us
    |- 1 : 1323 us
    |- 2 : 1318 us
  |- 200KB:
    |- 0 : 1467 us
    |- 1 : 1440 us
    |- 2 : 1562 us
  |- 500KB:
    |- 0 : 2205 us
    |- 1 : 2302 us
    |- 2 : 2106 us
  |- 1000KB:
    |- 0 : 3596 us
    |- 1 : 3518 us
    |- 2 : 3479 us

---RustSync---
  |- 10KB:
    |- 0 : 1342 us
    |- 1 : 1155 us
    |- 2 : 1175 us
  |- 50KB:
    |- 0 : 1273 us
    |- 1 : 1336 us
    |- 2 : 1417 us
  |- 100KB:
    |- 0 : 1379 us
    |- 1 : 1353 us
    |- 2 : 1330 us
  |- 200KB:
    |- 0 : 1596 us
    |- 1 : 1454 us
    |- 2 : 1600 us
  |- 500KB:
    |- 0 : 2194 us
    |- 1 : 2104 us
    |- 2 : 2165 us
  |- 1000KB:
    |- 0 : 3427 us
    |- 1 : 3332 us
    |- 2 : 3199 us