From 007ec769da70ee3133446653a71491185f008d40 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 28 Feb 2024 08:26:33 +0000 Subject: [PATCH 1/5] README: improve readme Signed-off-by: Ming Lei --- README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4a069b5..f9d4216 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Rust library for building linux ublk target device, which talks with linux `ublk driver`[^1] for exposing standard linux block device, -meantime all target IO logic can be moved to userspace. +meantime all target IO logic is implemented in userspace. Linux kernel 6.0 starts to support ublk covered by config option of CONFIG_BLK_DEV_UBLK. @@ -20,12 +20,9 @@ introduction](https://github.com/ming1/ubdsrv/blob/master/doc/ublk_intro.pdf) ## Quick Start -Follows one totally working 2-queue ublk-null target which is built over -libublk 0.1, and each queue depth is 64, and each IO\'s max buffer size -is 512KB. - -Ublk block device(/dev/ublkbN) is created after the code is run. And the -device will be deleted after terminating this process by ctrl+C. +Follows one 2-queue ublk-null target which is built over libublk, ublk block +device(/dev/ublkbN) is created after the code is run. And the device will be +deleted after terminating this process by ctrl+C. ``` rust use libublk::{ctrl::UblkCtrlBuilder, io::UblkDev, io::UblkQueue}; @@ -113,8 +110,15 @@ fn main() { } ``` - * [`examples/loop.rs`](examples/loop.rs): real example using async/await & io_uring. + * [`examples/loop.rs`](examples/loop.rs): real example using + async/await & io_uring. + + * [`examples/ramdisk.rs`](examples/ramdisk.rs): single thread & + async/.await for both ctrl and IO, this technique will be extended to + create multiple devices from single thread in future +`rublk`[^4] is based on libublk, and supports null, loop, zoned & qcow2 targets so +far. ## unprivileged ublk support @@ -171,3 +175,4 @@ Any kinds of contributions are welcome! [^1]: [^2]: [^3]: +[^4]: From 0f163e673f628f0b9137cb693908ac9edae6ea6e Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 28 Feb 2024 08:32:15 +0000 Subject: [PATCH 2/5] examples/loop: remove --split Now we have switched to smol executor, join!() works just fine, no necessary to cover --split test. Signed-off-by: Ming Lei --- .github/workflows/test.yml | 1 - examples/loop.rs | 61 ++------------------------------------ 2 files changed, 3 insertions(+), 59 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b9f5e6..77de758 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,6 @@ jobs: mkosi ssh /usr/share/libublk-rs/target/debug/test-basic --nocapture mkosi ssh truncate -s 128M /tmp/test.img mkosi ssh /usr/share/libublk-rs/target/debug/examples/loop add --foreground --oneshot -f /tmp/test.img -a - mkosi ssh /usr/share/libublk-rs/target/debug/examples/loop add --foreground --oneshot -f /tmp/test.img -a -s mkosi ssh /usr/share/libublk-rs/target/debug/examples/null add --foreground --oneshot -a mkosi ssh /usr/share/libublk-rs/target/debug/examples/null add --foreground --oneshot -a -u diff --git a/examples/loop.rs b/examples/loop.rs index 07ebc48..25293bb 100644 --- a/examples/loop.rs +++ b/examples/loop.rs @@ -31,7 +31,6 @@ bitflags! { struct LoFlags: u32 { const ASYNC = 0b00000001; const FOREGROUND = 0b00000010; - const SPLIT = 0b00000100; const ONESHOT = 0b00001000; } } @@ -83,7 +82,7 @@ fn lo_file_size(f: &std::fs::File) -> Result<(u64, u8, u8)> { } // setup loop target -fn lo_init_tgt(dev: &mut UblkDev, lo: &LoopTgt, split: bool) -> Result { +fn lo_init_tgt(dev: &mut UblkDev, lo: &LoopTgt) -> Result { trace!("loop: init_tgt {}", dev.dev_info.dev_id); if lo.direct_io != 0 { unsafe { @@ -96,10 +95,6 @@ fn lo_init_tgt(dev: &mut UblkDev, lo: &LoopTgt, split: bool) -> Result, tag: u16, buf_addr: *mut u8) return -libc::EAGAIN; } -async fn lo_handle_io_cmd_async_split(q: &UblkQueue<'_>, tag: u16, buf_addr: *mut u8) -> i32 { - let iod = q.get_iod(tag); - let res = __lo_prep_submit_io_cmd(iod); - if res < 0 { - return res; - } - - let op = iod.op_flags & 0xff; - let off = (iod.start_sector << 9) as u64; - let bytes = (iod.nr_sectors << 9) as u32; - - if bytes > 4096 { - let sqe = __lo_make_io_sqe(op, off, 4096, buf_addr); - let sqe2 = __lo_make_io_sqe( - op, - off + 4096, - bytes - 4096, - ((buf_addr as u64) + 4096) as *mut u8, - ); - - let f = q.ublk_submit_sqe(sqe); - let f2 = q.ublk_submit_sqe(sqe2); - let (res, res2) = futures::join!(f, f2); - - res + res2 - } else { - let sqe = __lo_make_io_sqe(op, off, bytes, buf_addr); - let f = q.ublk_submit_sqe(sqe); - - f.await - } -} - fn lo_handle_io_cmd_sync(q: &UblkQueue<'_>, tag: u16, i: &UblkIOCtx, buf_addr: *mut u8) { let iod = q.get_iod(tag); let op = iod.op_flags & 0xff; @@ -294,7 +256,6 @@ fn __test_add( lo_flags: LoFlags, ) { let aio = lo_flags.intersects(LoFlags::ASYNC); - let split = lo_flags.intersects(LoFlags::SPLIT); let oneshot = lo_flags.intersects(LoFlags::ONESHOT); { // LooTgt has to live in the whole device lifetime @@ -318,7 +279,7 @@ fn __test_add( .build() .unwrap(); - let tgt_init = |dev: &mut UblkDev| lo_init_tgt(dev, &lo, split); + let tgt_init = |dev: &mut UblkDev| lo_init_tgt(dev, &lo); let q_async_fn = move |qid: u16, dev: &UblkDev| { let q_rc = Rc::new(UblkQueue::new(qid as u16, &dev).unwrap()); let exe = smol::LocalExecutor::new(); @@ -340,11 +301,7 @@ fn __test_add( break; } - res = if !split { - lo_handle_io_cmd_async(&q, tag, buf_addr).await - } else { - lo_handle_io_cmd_async_split(&q, tag, buf_addr).await - }; + res = lo_handle_io_cmd_async(&q, tag, buf_addr).await; cmd_op = sys::UBLK_IO_COMMIT_AND_FETCH_REQ; } })); @@ -456,13 +413,6 @@ fn main() { .long("oneshot") .action(ArgAction::SetTrue) .help("create, dump and remove device automatically"), - ) - .arg( - Arg::new("split") - .long("split") - .short('s') - .action(ArgAction::SetTrue) - .help("Split big IO into two small IOs, only for --async"), ), ) .subcommand( @@ -506,11 +456,6 @@ fn main() { if add_matches.get_flag("async") { lo_flags |= LoFlags::ASYNC; }; - if lo_flags.intersects(LoFlags::ASYNC) { - if add_matches.get_flag("split") { - lo_flags |= LoFlags::SPLIT; - } - }; if add_matches.get_flag("foreground") { lo_flags |= LoFlags::FOREGROUND; }; From ebd95cb03610251ca204bb6ee29ee05e3dd614f1 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 29 Feb 2024 00:32:50 +0000 Subject: [PATCH 3/5] build: add three ioctl encoded command MARK_FIX_753(UBLK_U_IO_FETCH_REQ); MARK_FIX_753(UBLK_U_IO_COMMIT_AND_FETCH_REQ); MARK_FIX_753(UBLK_U_IO_NEED_GET_DATA); Prepare for supporting to send ioctl encoded command at default. Signed-off-by: Ming Lei --- build.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.rs b/build.rs index 365f449..30eef91 100644 --- a/build.rs +++ b/build.rs @@ -51,6 +51,9 @@ MARK_FIX_753(UBLK_U_CMD_START_USER_RECOVERY); MARK_FIX_753(UBLK_U_CMD_END_USER_RECOVERY); MARK_FIX_753(UBLK_U_CMD_GET_DEV_INFO2); MARK_FIX_753(UBLK_U_CMD_GET_FEATURES); +MARK_FIX_753(UBLK_U_IO_FETCH_REQ); +MARK_FIX_753(UBLK_U_IO_COMMIT_AND_FETCH_REQ); +MARK_FIX_753(UBLK_U_IO_NEED_GET_DATA); const int Fix753_UBLK_IO_RES_ABORT = UBLK_IO_RES_ABORT; "#; From d8dbedc934ad1ac2840e921f8f085ed692f1ff70 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 29 Feb 2024 00:28:49 +0000 Subject: [PATCH 4/5] tests: fix any test which enables USER_COPY Kernel may not support USER_COPY yet, so we have to deal with USER_COPY or !USER_COPY. Fix test failure on 6.1-stable kernel. Signed-off-by: Ming Lei --- examples/null.rs | 5 ++--- tests/basic.rs | 55 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/examples/null.rs b/examples/null.rs index e5ecb3f..5b93f9a 100755 --- a/examples/null.rs +++ b/examples/null.rs @@ -82,7 +82,6 @@ fn __test_add( let buf_addr = if user_copy { std::ptr::null_mut() } else { - let bufs = bufs_rc.clone(); bufs[tag as usize].as_mut_ptr() }; handle_io_cmd(q, tag, buf_addr); @@ -90,8 +89,8 @@ fn __test_add( UblkQueue::new(qid, dev) .unwrap() - .regiser_io_bufs(if user_copy { None } else { Some(&bufs) }) - .submit_fetch_commands(if user_copy { None } else { Some(&bufs) }) + .regiser_io_bufs(if user_copy { None } else { Some(&bufs_rc) }) + .submit_fetch_commands(if user_copy { None } else { Some(&bufs_rc) }) .wait_and_handle_io(io_handler); }; let q_async_handler = move |qid: u16, dev: &UblkDev| { diff --git a/tests/basic.rs b/tests/basic.rs index a8dca7f..c3f4594 100644 --- a/tests/basic.rs +++ b/tests/basic.rs @@ -77,18 +77,26 @@ mod integration { #[test] fn test_ublk_null() { /// called from queue_handler closure(), which supports Clone(), - fn null_handle_queue(qid: u16, _dev: &UblkDev) { + fn null_handle_queue(qid: u16, dev: &UblkDev) { + let bufs_rc = Rc::new(dev.alloc_queue_io_bufs()); + let user_copy = (dev.dev_info.flags & libublk::sys::UBLK_F_USER_COPY as u64) != 0; + let bufs = bufs_rc.clone(); + let io_handler = move |q: &UblkQueue, tag: u16, _io: &UblkIOCtx| { let iod = q.get_iod(tag); let bytes = (iod.nr_sectors << 9) as i32; - let buf_addr = std::ptr::null_mut(); + let buf_addr = if user_copy { + std::ptr::null_mut() + } else { + bufs[tag as usize].as_mut_ptr() + }; q.complete_io_cmd(tag, buf_addr, Ok(UblkIORes::Result(bytes))); }; - UblkQueue::new(qid, _dev) + UblkQueue::new(qid, dev) .unwrap() - .submit_fetch_commands(None) + .submit_fetch_commands(if user_copy { None } else { Some(&bufs_rc) }) .wait_and_handle_io(io_handler); } @@ -101,19 +109,28 @@ mod integration { fn test_ublk_null_comp_batch() { use libublk::UblkFatRes; /// called from queue_handler closure(), which supports Clone(), - fn null_handle_queue_batch(qid: u16, _dev: &UblkDev) { + fn null_handle_queue_batch(qid: u16, dev: &UblkDev) { + let bufs_rc = Rc::new(dev.alloc_queue_io_bufs()); + let user_copy = (dev.dev_info.flags & libublk::sys::UBLK_F_USER_COPY as u64) != 0; + let bufs = bufs_rc.clone(); + let io_handler = move |q: &UblkQueue, tag: u16, _io: &UblkIOCtx| { let iod = q.get_iod(tag); let bytes = (iod.nr_sectors << 9) as i32; - let buf_addr = std::ptr::null_mut(); + + let buf_addr = if user_copy { + std::ptr::null_mut() + } else { + bufs[tag as usize].as_mut_ptr() + }; let res = Ok(UblkIORes::FatRes(UblkFatRes::BatchRes(vec![(tag, bytes)]))); q.complete_io_cmd(tag, buf_addr, res); }; - UblkQueue::new(qid, _dev) + UblkQueue::new(qid, dev) .unwrap() - .submit_fetch_commands(None) + .submit_fetch_commands(if user_copy { None } else { Some(&bufs_rc) }) .wait_and_handle_io(io_handler); } @@ -303,7 +320,6 @@ mod integration { let bufs = bufs_rc.clone(); let io_handler = move |q: &UblkQueue, tag: u16, _io: &UblkIOCtx| { - let bufs = bufs_rc.clone(); let buf_addr = bufs[tag as usize].as_mut_ptr(); rd_handle_io(q, tag, _io, buf_addr, dev_addr); @@ -311,8 +327,8 @@ mod integration { UblkQueue::new(qid, dev) .unwrap() - .regiser_io_bufs(Some(&bufs)) - .submit_fetch_commands(Some(&bufs)) + .regiser_io_bufs(Some(&bufs_rc)) + .submit_fetch_commands(Some(&bufs_rc)) .wait_and_handle_io(io_handler); }; @@ -326,13 +342,16 @@ mod integration { #[test] fn test_fn_mut_io_closure() { /// called from queue_handler closure(), which supports Clone(), - fn null_queue_mut_io(qid: u16, _dev: &UblkDev) { + fn null_queue_mut_io(qid: u16, dev: &UblkDev) { + let bufs_rc = Rc::new(dev.alloc_queue_io_bufs()); + let user_copy = (dev.dev_info.flags & libublk::sys::UBLK_F_USER_COPY as u64) != 0; + let bufs = bufs_rc.clone(); + // modify this vector in io handling closure let mut q_vec = Vec::::new(); let io_handler = move |q: &UblkQueue, tag: u16, _io: &UblkIOCtx| { let iod = q.get_iod(tag); let res = Ok(UblkIORes::Result((iod.nr_sectors << 9) as i32)); - let buf_addr = std::ptr::null_mut(); { q_vec.push(tag as i32); @@ -341,12 +360,18 @@ mod integration { } } + let buf_addr = if user_copy { + std::ptr::null_mut() + } else { + let bufs = bufs_rc.clone(); + bufs[tag as usize].as_mut_ptr() + }; q.complete_io_cmd(tag, buf_addr, res); }; - UblkQueue::new(qid, _dev) + UblkQueue::new(qid, dev) .unwrap() - .submit_fetch_commands(None) + .submit_fetch_commands(if user_copy { None } else { Some(&bufs) }) .wait_and_handle_io(io_handler); } From a9440623f481d10c453ee9183d15ac4b78727d01 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 28 Feb 2024 16:28:38 +0000 Subject: [PATCH 5/5] libublk: support to send command via ioctl encode Otherwise, libublk may not work in case that BLKDEV_UBLK_LEGACY_OPCODES is off. Signed-off-by: Ming Lei --- README.md | 4 +- examples/loop.rs | 4 +- examples/null.rs | 4 +- examples/ramdisk.rs | 4 +- src/ctrl.rs | 115 ++++++++++++++++++++++++++++++++++---------- src/io.rs | 36 +++++++++++--- src/lib.rs | 3 ++ tests/basic.rs | 4 +- 8 files changed, 132 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index f9d4216..709b2b6 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ async fn io_task(q: &UblkQueue<'_>, tag: u16) { // IO buffer for exchange data with /dev/ublkbN let buf_bytes = q.dev.dev_info.max_io_buf_bytes as usize; let buf = libublk::helpers::IoBuf::::new(buf_bytes); - let mut cmd_op = libublk::sys::UBLK_IO_FETCH_REQ; + let mut cmd_op = libublk::sys::UBLK_U_IO_FETCH_REQ; let mut res = 0; // Register IO buffer, so that buffer pages can be discarded @@ -53,7 +53,7 @@ async fn io_task(q: &UblkQueue<'_>, tag: u16) { // Handle this incoming IO command res = handle_io_cmd(&q, tag).await; - cmd_op = libublk::sys::UBLK_IO_COMMIT_AND_FETCH_REQ; + cmd_op = libublk::sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ; } } diff --git a/examples/loop.rs b/examples/loop.rs index 25293bb..88ba8f8 100644 --- a/examples/loop.rs +++ b/examples/loop.rs @@ -291,7 +291,7 @@ fn __test_add( f_vec.push(exe.spawn(async move { let buf = IoBuf::::new(q.dev.dev_info.max_io_buf_bytes as usize); let buf_addr = buf.as_mut_ptr(); - let mut cmd_op = sys::UBLK_IO_FETCH_REQ; + let mut cmd_op = sys::UBLK_U_IO_FETCH_REQ; let mut res = 0; q.register_io_buf(tag, &buf); @@ -302,7 +302,7 @@ fn __test_add( } res = lo_handle_io_cmd_async(&q, tag, buf_addr).await; - cmd_op = sys::UBLK_IO_COMMIT_AND_FETCH_REQ; + cmd_op = sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ; } })); } diff --git a/examples/null.rs b/examples/null.rs index 5b93f9a..b1a0df7 100755 --- a/examples/null.rs +++ b/examples/null.rs @@ -103,7 +103,7 @@ fn __test_add( f_vec.push(exe.spawn(async move { let buf = IoBuf::::new(q.dev.dev_info.max_io_buf_bytes as usize); - let mut cmd_op = libublk::sys::UBLK_IO_FETCH_REQ; + let mut cmd_op = libublk::sys::UBLK_U_IO_FETCH_REQ; let mut res = 0; let buf_addr = if user_copy { std::ptr::null_mut() @@ -119,7 +119,7 @@ fn __test_add( } res = get_io_cmd_result(&q, tag); - cmd_op = libublk::sys::UBLK_IO_COMMIT_AND_FETCH_REQ; + cmd_op = libublk::sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ; } })); } diff --git a/examples/ramdisk.rs b/examples/ramdisk.rs index e877a64..3b63e5b 100644 --- a/examples/ramdisk.rs +++ b/examples/ramdisk.rs @@ -60,7 +60,7 @@ fn queue_fn<'a>( f_vec.push(exe.spawn(async move { let buffer = IoBuf::::new(buf_size); let addr = buffer.as_mut_ptr(); - let mut cmd_op = libublk::sys::UBLK_IO_FETCH_REQ; + let mut cmd_op = libublk::sys::UBLK_U_IO_FETCH_REQ; let mut res = 0; loop { @@ -70,7 +70,7 @@ fn queue_fn<'a>( } res = handle_io(&q, tag, addr, dev_buf_addr); - cmd_op = libublk::sys::UBLK_IO_COMMIT_AND_FETCH_REQ; + cmd_op = libublk::sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ; } })); exe.try_tick(); diff --git a/src/ctrl.rs b/src/ctrl.rs index 30544a7..54f8c1b 100644 --- a/src/ctrl.rs +++ b/src/ctrl.rs @@ -88,7 +88,9 @@ struct UblkCtrlCmdData { impl UblkCtrlCmdData { fn prep_un_privileged_dev_path(&mut self, dev: &UblkCtrlInner) -> (u64, Option>) { // handle GET_DEV_INFO2 always with dev_path attached - if self.cmd_op != sys::UBLK_CMD_GET_DEV_INFO2 + let cmd_op = self.cmd_op & 0xff; + + if cmd_op != sys::UBLK_CMD_GET_DEV_INFO2 && (!dev.is_unprivileged() || (self.flags & CTRL_CMD_NO_NEED_DEV_PATH) != 0) { return (0, None); @@ -136,7 +138,9 @@ impl UblkCtrlCmdData { } fn unprep_un_privileged_dev_path(&mut self, dev: &UblkCtrlInner, buf: u64) { - if self.cmd_op != sys::UBLK_CMD_GET_DEV_INFO2 + let cmd_op = self.cmd_op & 0xff; + + if cmd_op != sys::UBLK_CMD_GET_DEV_INFO2 && (!dev.is_unprivileged() || (self.flags & CTRL_CMD_NO_NEED_DEV_PATH) != 0) { return; @@ -364,7 +368,12 @@ impl UblkCtrlInner { } dev.read_dev_info()?; } - trace!("ctrl: device {} created", dev.dev_info.dev_id); + + trace!( + "ctrl: device {} flags {:x} created", + dev.dev_info.dev_id, + dev.dev_flags + ); Ok(dev) } @@ -532,29 +541,32 @@ impl UblkCtrlInner { /// check one control command and see if it is completed /// - fn poll_cmd(&mut self, token: u64) -> Result { + fn poll_cmd(&mut self, token: u64) -> i32 { CTRL_URING.with(|refcell| { let mut r = refcell.borrow_mut(); if r.completion().is_empty() { - Err(UblkError::UringIOError(-libc::EAGAIN)) + //Err(UblkError::UringIOError(-libc::EAGAIN)) + -libc::EAGAIN } else { let cqe = r.completion().next().expect("cqueue is empty"); if cqe.user_data() != token { - Err(UblkError::UringIOError(-libc::EAGAIN)) + //Err(UblkError::UringIOError(-libc::EAGAIN)) + -libc::EAGAIN } else { let res: i32 = cqe.result(); if res == 0 || res == -libc::EBUSY { - Ok(res) + res } else { - Err(UblkError::UringIOError(res)) + //Err(UblkError::UringIOError(res)) + res } } } }) } - async fn ublk_ctrl_cmd_async(&mut self, data: &UblkCtrlCmdData) -> Result { + async fn __ublk_ctrl_cmd_async(&mut self, data: &UblkCtrlCmdData) -> i32 { let mut data = *data; let (old_buf, _new) = data.prep_un_privileged_dev_path(self); @@ -562,15 +574,41 @@ impl UblkCtrlInner { data.unprep_un_privileged_dev_path(self, old_buf); - Ok(res) + res } - fn ublk_ctrl_cmd(&mut self, data: &UblkCtrlCmdData) -> Result { + async fn ublk_ctrl_cmd_async(&mut self, data: &UblkCtrlCmdData) -> Result { + let res = self.__ublk_ctrl_cmd_async(data).await; + + if res >= 0 || res == -libc::EBUSY { + Ok(res) + } else { + let legacy_op = data.cmd_op & 0xff; + + // new commands have to be issued via ioctl encoding + if legacy_op > sys::UBLK_CMD_GET_DEV_INFO2 { + Err(UblkError::UringIOError(res)) + } else { + let mut new_data = *data; + + // retry one more time with legacy encoding + new_data.cmd_op = legacy_op; + let res = self.__ublk_ctrl_cmd_async(&new_data).await; + if res >= 0 || res == -libc::EBUSY { + Ok(res) + } else { + Err(UblkError::UringIOError(res)) + } + } + } + } + + fn __ublk_ctrl_cmd(&mut self, data: &UblkCtrlCmdData) -> i32 { let mut data = *data; let to_wait = 1; let (old_buf, _new) = data.prep_un_privileged_dev_path(self); - let token = self.ublk_submit_cmd(&data, to_wait)?; + let token = self.ublk_submit_cmd(&data, to_wait).unwrap(); let res = self.poll_cmd(token); data.unprep_un_privileged_dev_path(self, old_buf); @@ -578,9 +616,36 @@ impl UblkCtrlInner { res } + fn ublk_ctrl_cmd(&mut self, data: &UblkCtrlCmdData) -> Result { + let res = self.__ublk_ctrl_cmd(data); + + if res >= 0 || res == -libc::EBUSY { + Ok(res) + } else { + let legacy_op = data.cmd_op & 0xff; + + // new commands have to be issued via ioctl encoding + if legacy_op > sys::UBLK_CMD_GET_DEV_INFO2 { + Err(UblkError::UringIOError(res)) + } else { + let mut new_data = *data; + + // retry one more time with legacy encoding for + // old kernel without ioctl encoding support + new_data.cmd_op = legacy_op; + let res = self.__ublk_ctrl_cmd(&new_data); + if res >= 0 || res == -libc::EBUSY { + Ok(res) + } else { + Err(UblkError::UringIOError(res)) + } + } + } + } + fn add(&mut self) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_ADD_DEV, + cmd_op: sys::UBLK_U_CMD_ADD_DEV, flags: CTRL_CMD_HAS_BUF | CTRL_CMD_NO_NEED_DEV_PATH, addr: std::ptr::addr_of!(self.dev_info) as u64, len: core::mem::size_of::() as u32, @@ -594,7 +659,7 @@ impl UblkCtrlInner { /// fn del(&mut self) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_DEL_DEV, + cmd_op: sys::UBLK_U_CMD_DEL_DEV, ..Default::default() }; @@ -618,7 +683,7 @@ impl UblkCtrlInner { fn __read_dev_info(&mut self) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_GET_DEV_INFO, + cmd_op: sys::UBLK_U_CMD_GET_DEV_INFO, flags: CTRL_CMD_HAS_BUF | CTRL_CMD_BUF_READ, addr: std::ptr::addr_of!(self.dev_info) as u64, len: core::mem::size_of::() as u32, @@ -630,7 +695,7 @@ impl UblkCtrlInner { fn __read_dev_info2(&mut self) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_GET_DEV_INFO2, + cmd_op: sys::UBLK_U_CMD_GET_DEV_INFO2, flags: CTRL_CMD_HAS_BUF | CTRL_CMD_BUF_READ, addr: std::ptr::addr_of!(self.dev_info) as u64, len: core::mem::size_of::() as u32, @@ -654,7 +719,7 @@ impl UblkCtrlInner { /// fn start(&mut self, pid: i32) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_START_DEV, + cmd_op: sys::UBLK_U_CMD_START_DEV, flags: CTRL_CMD_HAS_DATA, data: pid as u64, ..Default::default() @@ -667,7 +732,7 @@ impl UblkCtrlInner { /// async fn start_async(&mut self, pid: i32) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_START_DEV, + cmd_op: sys::UBLK_U_CMD_START_DEV, flags: CTRL_CMD_HAS_DATA, data: pid as u64, ..Default::default() @@ -680,7 +745,7 @@ impl UblkCtrlInner { /// fn stop(&mut self) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_STOP_DEV, + cmd_op: sys::UBLK_U_CMD_STOP_DEV, ..Default::default() }; @@ -694,7 +759,7 @@ impl UblkCtrlInner { fn get_params(&mut self, params: &mut sys::ublk_params) -> Result { params.len = core::mem::size_of::() as u32; let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_GET_PARAMS, + cmd_op: sys::UBLK_U_CMD_GET_PARAMS, flags: CTRL_CMD_HAS_BUF | CTRL_CMD_BUF_READ, addr: params as *const sys::ublk_params as u64, len: params.len, @@ -713,7 +778,7 @@ impl UblkCtrlInner { p.len = core::mem::size_of::() as u32; let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_SET_PARAMS, + cmd_op: sys::UBLK_U_CMD_SET_PARAMS, flags: CTRL_CMD_HAS_BUF, addr: std::ptr::addr_of!(p) as u64, len: p.len, @@ -725,7 +790,7 @@ impl UblkCtrlInner { fn get_queue_affinity(&mut self, q: u32, bm: &mut UblkQueueAffinity) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_GET_QUEUE_AFFINITY, + cmd_op: sys::UBLK_U_CMD_GET_QUEUE_AFFINITY, flags: CTRL_CMD_HAS_BUF | CTRL_CMD_HAS_DATA | CTRL_CMD_BUF_READ, addr: bm.addr() as u64, data: q as u64, @@ -737,7 +802,7 @@ impl UblkCtrlInner { fn __start_user_recover(&mut self) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_START_USER_RECOVERY, + cmd_op: sys::UBLK_U_CMD_START_USER_RECOVERY, ..Default::default() }; @@ -748,7 +813,7 @@ impl UblkCtrlInner { /// fn end_user_recover(&mut self, pid: i32) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_END_USER_RECOVERY, + cmd_op: sys::UBLK_U_CMD_END_USER_RECOVERY, flags: CTRL_CMD_HAS_DATA, data: pid as u64, ..Default::default() @@ -761,7 +826,7 @@ impl UblkCtrlInner { /// async fn end_user_recover_async(&mut self, pid: i32) -> Result { let data: UblkCtrlCmdData = UblkCtrlCmdData { - cmd_op: sys::UBLK_CMD_END_USER_RECOVERY, + cmd_op: sys::UBLK_U_CMD_END_USER_RECOVERY, flags: CTRL_CMD_HAS_DATA, data: pid as u64, ..Default::default() diff --git a/src/io.rs b/src/io.rs index bffffee..f5d0389 100644 --- a/src/io.rs +++ b/src/io.rs @@ -119,8 +119,9 @@ impl<'a> UblkIOCtx<'a> { #[inline(always)] #[allow(arithmetic_overflow)] pub fn build_user_data(tag: u16, op: u32, tgt_data: u32, is_target_io: bool) -> u64 { - assert!((op >> 8) == 0 && (tgt_data >> 16) == 0); + assert!((tgt_data >> 16) == 0); + let op = op & 0xff; tag as u64 | (op << 16) as u64 | (tgt_data << 24) as u64 | ((is_target_io as u64) << 63) } @@ -465,6 +466,8 @@ fn round_up(val: u32, rnd: u32) -> u32 { impl UblkQueue<'_> { const UBLK_QUEUE_IDLE_SECS: u32 = 20; + const UBLK_QUEUE_IOCTL_ENCODE: u32 = crate::dev_flags::UBLK_DEV_F_INTERNAL_0; + #[inline(always)] fn cmd_buf_sz(depth: u32) -> u32 { let size = depth * core::mem::size_of::() as u32; @@ -473,6 +476,11 @@ impl UblkQueue<'_> { round_up(size, page_sz) } + #[inline(always)] + fn is_ioctl_encode(&self) -> bool { + (self.flags & Self::UBLK_QUEUE_IOCTL_ENCODE) != 0 + } + /// New one ublk queue /// /// # Arguments: @@ -532,8 +540,15 @@ impl UblkQueue<'_> { bufs[i as usize] = std::ptr::null_mut(); } + assert!((dev.flags & Self::UBLK_QUEUE_IOCTL_ENCODE) == 0); + let q = UblkQueue { - flags: dev.flags, + flags: dev.flags + | if (dev.dev_info.flags & (sys::UBLK_F_CMD_IOCTL_ENCODE as u64)) != 0 { + Self::UBLK_QUEUE_IOCTL_ENCODE + } else { + 0 + }, q_id, q_depth: depth, io_cmd_buf: io_cmd_buf as u64, @@ -647,6 +662,12 @@ impl UblkQueue<'_> { result: res, }; + let cmd_op = if !self.is_ioctl_encode() { + cmd_op & 0xff + } else { + cmd_op + }; + let sqe = opcode::UringCmd16::new(types::Fixed(0), cmd_op) .cmd(unsafe { core::mem::transmute::(io_cmd) }) .build() @@ -667,9 +688,10 @@ impl UblkQueue<'_> { state.inc_cmd_inflight(); log::trace!( - "{}: (qid {} tag {} cmd_op {}) stopping {}", + "{}: (qid {} flags {:x} tag {} cmd_op {}) stopping {}", "queue_io_cmd", self.q_id, + self.flags, tag, cmd_op, state.is_stopping(), @@ -702,7 +724,7 @@ impl UblkQueue<'_> { self.queue_io_cmd( r, tag, - sys::UBLK_IO_COMMIT_AND_FETCH_REQ, + sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ, buf_addr, io_cmd_result, ); @@ -711,7 +733,7 @@ impl UblkQueue<'_> { /// Submit one io command. /// /// When it is called 1st time on this tag, the `cmd_op` has to be - /// UBLK_IO_FETCH_REQ, otherwise it is UBLK_IO_COMMIT_AND_FETCH_REQ. + /// UBLK_U_IO_FETCH_REQ, otherwise it is UBLK_U_IO_COMMIT_AND_FETCH_REQ. /// /// UblkUringOpFuture is one Future object, so this function is actually /// one async function, and user can get result by submit_io_cmd().await @@ -776,7 +798,7 @@ impl UblkQueue<'_> { self.queue_io_cmd( &mut self.q_ring.borrow_mut(), i as u16, - sys::UBLK_IO_FETCH_REQ, + sys::UBLK_U_IO_FETCH_REQ, buf_addr as u64, -1, ); @@ -789,7 +811,7 @@ impl UblkQueue<'_> { self.queue_io_cmd( &mut self.q_ring.borrow_mut(), i as u16, - sys::UBLK_IO_FETCH_REQ, + sys::UBLK_U_IO_FETCH_REQ, buf_addr, -1, ); diff --git a/src/lib.rs b/src/lib.rs index 3e59fc6..214774e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ pub mod io; pub mod sys; pub mod uring_async; +/// Don't use the top 8 bits, which are reserved for internal uses pub mod dev_flags { /// feature: support IO batch completion from single IO tag, typical /// usecase is to complete IOs from eventfd CQE handler @@ -24,6 +25,8 @@ pub mod dev_flags { /// tell UblkCtrl that we are recovering one old device pub const UBLK_DEV_F_RECOVER_DEV: u32 = 1u32 << 2; + pub(crate) const UBLK_DEV_F_INTERNAL_0: u32 = 1u32 << 31; + pub const UBLK_DEV_F_ALL: u32 = UBLK_DEV_F_COMP_BATCH | UBLK_DEV_F_ADD_DEV | UBLK_DEV_F_RECOVER_DEV; } diff --git a/tests/basic.rs b/tests/basic.rs index c3f4594..8ba0f3b 100644 --- a/tests/basic.rs +++ b/tests/basic.rs @@ -196,7 +196,7 @@ mod integration { let __dev_data = _dev_data.clone(); f_vec.push(exe.spawn(async move { - let mut cmd_op = sys::UBLK_IO_FETCH_REQ; + let mut cmd_op = sys::UBLK_U_IO_FETCH_REQ; let buf = IoBuf::::new(q.dev.dev_info.max_io_buf_bytes as usize); let mut res = 0; @@ -208,7 +208,7 @@ mod integration { } res = handle_io_cmd(&q, tag).await; - cmd_op = sys::UBLK_IO_COMMIT_AND_FETCH_REQ; + cmd_op = sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ; { let mut guard = __dev_data.lock().unwrap(); (*guard).done += 1;