Merge pull request #8 from ublk-org/ublk-ioctl-encode

Ublk ioctl encode: send ioctl encoded op code at default
ublk-org · Feb 29, 2024 · 1aa3743 · 1aa3743
2 parents 78de4a8 + a944062
commit 1aa3743
Show file tree

Hide file tree

Showing 10 changed files with 193 additions and 127 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -78,7 +78,6 @@ jobs:
         mkosi ssh /usr/share/libublk-rs/target/debug/test-basic --nocapture
         mkosi ssh truncate -s 128M /tmp/test.img
         mkosi ssh /usr/share/libublk-rs/target/debug/examples/loop add --foreground --oneshot -f /tmp/test.img -a
-        mkosi ssh /usr/share/libublk-rs/target/debug/examples/loop add --foreground --oneshot -f /tmp/test.img -a -s
         mkosi ssh /usr/share/libublk-rs/target/debug/examples/null add --foreground --oneshot -a
         mkosi ssh /usr/share/libublk-rs/target/debug/examples/null add --foreground --oneshot -a -u
 

diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 
 Rust library for building linux ublk target device, which talks with
 linux `ublk driver`[^1] for exposing standard linux block device,
-meantime all target IO logic can be moved to userspace.
+meantime all target IO logic is implemented in userspace.
 
 Linux kernel 6.0 starts to support ublk covered by config option of
 CONFIG_BLK_DEV_UBLK.
@@ -20,12 +20,9 @@ introduction](https://github.com/ming1/ubdsrv/blob/master/doc/ublk_intro.pdf)
 
 ## Quick Start
 
-Follows one totally working 2-queue ublk-null target which is built over
-libublk 0.1, and each queue depth is 64, and each IO\'s max buffer size
-is 512KB.
-
-Ublk block device(/dev/ublkbN) is created after the code is run. And the
-device will be deleted after terminating this process by ctrl+C.
+Follows one 2-queue ublk-null target which is built over libublk, ublk block
+device(/dev/ublkbN) is created after the code is run. And the device will be
+deleted after terminating this process by ctrl+C.
 
 ``` rust
 use libublk::{ctrl::UblkCtrlBuilder, io::UblkDev, io::UblkQueue};
@@ -40,7 +37,7 @@ async fn io_task(q: &UblkQueue<'_>, tag: u16) {
     // IO buffer for exchange data with /dev/ublkbN
     let buf_bytes = q.dev.dev_info.max_io_buf_bytes as usize;
     let buf = libublk::helpers::IoBuf::<u8>::new(buf_bytes);
-    let mut cmd_op = libublk::sys::UBLK_IO_FETCH_REQ;
+    let mut cmd_op = libublk::sys::UBLK_U_IO_FETCH_REQ;
     let mut res = 0;
 
     // Register IO buffer, so that buffer pages can be discarded
@@ -56,7 +53,7 @@ async fn io_task(q: &UblkQueue<'_>, tag: u16) {
 
         // Handle this incoming IO command
         res = handle_io_cmd(&q, tag).await;
-        cmd_op = libublk::sys::UBLK_IO_COMMIT_AND_FETCH_REQ;
+        cmd_op = libublk::sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ;
     }
 }
 
@@ -113,8 +110,15 @@ fn main() {
 }
 ```
 
- * [`examples/loop.rs`](examples/loop.rs): real example using async/await & io_uring.
+ * [`examples/loop.rs`](examples/loop.rs): real example using
+   async/await & io_uring.
+
+ * [`examples/ramdisk.rs`](examples/ramdisk.rs): single thread &
+   async/.await for both ctrl and IO, this technique will be extended to
+   create multiple devices from single thread in future
 
+`rublk`[^4] is based on libublk, and supports null, loop, zoned & qcow2 targets so
+far.
 
 ## unprivileged ublk support
 
@@ -171,3 +175,4 @@ Any kinds of contributions are welcome!
 [^1]: <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/block/ublk_drv.c?h=v6.0>
 [^2]: <https://github.com/axboe/fio/blob/master/t/io_uring.c>
 [^3]: <https://github.com/osandov/blktests/blob/master/src/miniublk.c>
+[^4]: <https://github.com/ublk-org/rublk>
diff --git a/build.rs b/build.rs
@@ -51,6 +51,9 @@ MARK_FIX_753(UBLK_U_CMD_START_USER_RECOVERY);
 MARK_FIX_753(UBLK_U_CMD_END_USER_RECOVERY);
 MARK_FIX_753(UBLK_U_CMD_GET_DEV_INFO2);
 MARK_FIX_753(UBLK_U_CMD_GET_FEATURES);
+MARK_FIX_753(UBLK_U_IO_FETCH_REQ);
+MARK_FIX_753(UBLK_U_IO_COMMIT_AND_FETCH_REQ);
+MARK_FIX_753(UBLK_U_IO_NEED_GET_DATA);
 const int Fix753_UBLK_IO_RES_ABORT = UBLK_IO_RES_ABORT;
     "#;
 

diff --git a/examples/loop.rs b/examples/loop.rs
@@ -31,7 +31,6 @@ bitflags! {
     struct LoFlags: u32 {
         const ASYNC = 0b00000001;
         const FOREGROUND = 0b00000010;
-        const SPLIT = 0b00000100;
         const ONESHOT = 0b00001000;
     }
 }
@@ -83,7 +82,7 @@ fn lo_file_size(f: &std::fs::File) -> Result<(u64, u8, u8)> {
 }
 
 // setup loop target
-fn lo_init_tgt(dev: &mut UblkDev, lo: &LoopTgt, split: bool) -> Result<i32, UblkError> {
+fn lo_init_tgt(dev: &mut UblkDev, lo: &LoopTgt) -> Result<i32, UblkError> {
     trace!("loop: init_tgt {}", dev.dev_info.dev_id);
     if lo.direct_io != 0 {
         unsafe {
@@ -96,10 +95,6 @@ fn lo_init_tgt(dev: &mut UblkDev, lo: &LoopTgt, split: bool) -> Result<i32, Ublk
     tgt.fds[nr_fds as usize] = lo.back_file.as_raw_fd();
     tgt.nr_fds = nr_fds + 1;
 
-    let depth = dev.dev_info.queue_depth;
-    tgt.sq_depth = if split { depth * 2 } else { depth };
-    tgt.cq_depth = if split { depth * 2 } else { depth };
-
     let sz = { lo_file_size(&lo.back_file).unwrap() };
     tgt.dev_size = sz.0;
     //todo: figure out correct block size
@@ -176,39 +171,6 @@ async fn lo_handle_io_cmd_async(q: &UblkQueue<'_>, tag: u16, buf_addr: *mut u8)
     return -libc::EAGAIN;
 }
 
-async fn lo_handle_io_cmd_async_split(q: &UblkQueue<'_>, tag: u16, buf_addr: *mut u8) -> i32 {
-    let iod = q.get_iod(tag);
-    let res = __lo_prep_submit_io_cmd(iod);
-    if res < 0 {
-        return res;
-    }
-
-    let op = iod.op_flags & 0xff;
-    let off = (iod.start_sector << 9) as u64;
-    let bytes = (iod.nr_sectors << 9) as u32;
-
-    if bytes > 4096 {
-        let sqe = __lo_make_io_sqe(op, off, 4096, buf_addr);
-        let sqe2 = __lo_make_io_sqe(
-            op,
-            off + 4096,
-            bytes - 4096,
-            ((buf_addr as u64) + 4096) as *mut u8,
-        );
-
-        let f = q.ublk_submit_sqe(sqe);
-        let f2 = q.ublk_submit_sqe(sqe2);
-        let (res, res2) = futures::join!(f, f2);
-
-        res + res2
-    } else {
-        let sqe = __lo_make_io_sqe(op, off, bytes, buf_addr);
-        let f = q.ublk_submit_sqe(sqe);
-
-        f.await
-    }
-}
-
 fn lo_handle_io_cmd_sync(q: &UblkQueue<'_>, tag: u16, i: &UblkIOCtx, buf_addr: *mut u8) {
     let iod = q.get_iod(tag);
     let op = iod.op_flags & 0xff;
@@ -294,7 +256,6 @@ fn __test_add(
     lo_flags: LoFlags,
 ) {
     let aio = lo_flags.intersects(LoFlags::ASYNC);
-    let split = lo_flags.intersects(LoFlags::SPLIT);
     let oneshot = lo_flags.intersects(LoFlags::ONESHOT);
     {
         // LooTgt has to live in the whole device lifetime
@@ -318,7 +279,7 @@ fn __test_add(
             .build()
             .unwrap();
 
-        let tgt_init = |dev: &mut UblkDev| lo_init_tgt(dev, &lo, split);
+        let tgt_init = |dev: &mut UblkDev| lo_init_tgt(dev, &lo);
         let q_async_fn = move |qid: u16, dev: &UblkDev| {
             let q_rc = Rc::new(UblkQueue::new(qid as u16, &dev).unwrap());
             let exe = smol::LocalExecutor::new();
@@ -330,7 +291,7 @@ fn __test_add(
                 f_vec.push(exe.spawn(async move {
                     let buf = IoBuf::<u8>::new(q.dev.dev_info.max_io_buf_bytes as usize);
                     let buf_addr = buf.as_mut_ptr();
-                    let mut cmd_op = sys::UBLK_IO_FETCH_REQ;
+                    let mut cmd_op = sys::UBLK_U_IO_FETCH_REQ;
                     let mut res = 0;
 
                     q.register_io_buf(tag, &buf);
@@ -340,12 +301,8 @@ fn __test_add(
                             break;
                         }
 
-                        res = if !split {
-                            lo_handle_io_cmd_async(&q, tag, buf_addr).await
-                        } else {
-                            lo_handle_io_cmd_async_split(&q, tag, buf_addr).await
-                        };
-                        cmd_op = sys::UBLK_IO_COMMIT_AND_FETCH_REQ;
+                        res = lo_handle_io_cmd_async(&q, tag, buf_addr).await;
+                        cmd_op = sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ;
                     }
                 }));
             }
@@ -456,13 +413,6 @@ fn main() {
                         .long("oneshot")
                         .action(ArgAction::SetTrue)
                         .help("create, dump and remove device automatically"),
-                )
-                .arg(
-                    Arg::new("split")
-                        .long("split")
-                        .short('s')
-                        .action(ArgAction::SetTrue)
-                        .help("Split big IO into two small IOs, only for --async"),
                 ),
         )
         .subcommand(
@@ -506,11 +456,6 @@ fn main() {
             if add_matches.get_flag("async") {
                 lo_flags |= LoFlags::ASYNC;
             };
-            if lo_flags.intersects(LoFlags::ASYNC) {
-                if add_matches.get_flag("split") {
-                    lo_flags |= LoFlags::SPLIT;
-                }
-            };
             if add_matches.get_flag("foreground") {
                 lo_flags |= LoFlags::FOREGROUND;
             };

diff --git a/examples/null.rs b/examples/null.rs
@@ -82,16 +82,15 @@ fn __test_add(
                 let buf_addr = if user_copy {
                     std::ptr::null_mut()
                 } else {
-                    let bufs = bufs_rc.clone();
                     bufs[tag as usize].as_mut_ptr()
                 };
                 handle_io_cmd(q, tag, buf_addr);
             };
 
             UblkQueue::new(qid, dev)
                 .unwrap()
-                .regiser_io_bufs(if user_copy { None } else { Some(&bufs) })
-                .submit_fetch_commands(if user_copy { None } else { Some(&bufs) })
+                .regiser_io_bufs(if user_copy { None } else { Some(&bufs_rc) })
+                .submit_fetch_commands(if user_copy { None } else { Some(&bufs_rc) })
                 .wait_and_handle_io(io_handler);
         };
         let q_async_handler = move |qid: u16, dev: &UblkDev| {
@@ -104,7 +103,7 @@ fn __test_add(
 
                 f_vec.push(exe.spawn(async move {
                     let buf = IoBuf::<u8>::new(q.dev.dev_info.max_io_buf_bytes as usize);
-                    let mut cmd_op = libublk::sys::UBLK_IO_FETCH_REQ;
+                    let mut cmd_op = libublk::sys::UBLK_U_IO_FETCH_REQ;
                     let mut res = 0;
                     let buf_addr = if user_copy {
                         std::ptr::null_mut()
@@ -120,7 +119,7 @@ fn __test_add(
                         }
 
                         res = get_io_cmd_result(&q, tag);
-                        cmd_op = libublk::sys::UBLK_IO_COMMIT_AND_FETCH_REQ;
+                        cmd_op = libublk::sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ;
                     }
                 }));
             }

diff --git a/examples/ramdisk.rs b/examples/ramdisk.rs
@@ -60,7 +60,7 @@ fn queue_fn<'a>(
         f_vec.push(exe.spawn(async move {
             let buffer = IoBuf::<u8>::new(buf_size);
             let addr = buffer.as_mut_ptr();
-            let mut cmd_op = libublk::sys::UBLK_IO_FETCH_REQ;
+            let mut cmd_op = libublk::sys::UBLK_U_IO_FETCH_REQ;
             let mut res = 0;
 
             loop {
@@ -70,7 +70,7 @@ fn queue_fn<'a>(
                 }
 
                 res = handle_io(&q, tag, addr, dev_buf_addr);
-                cmd_op = libublk::sys::UBLK_IO_COMMIT_AND_FETCH_REQ;
+                cmd_op = libublk::sys::UBLK_U_IO_COMMIT_AND_FETCH_REQ;
             }
         }));
         exe.try_tick();