Refine select nodelay (#211)

* rewrite FdSet and it's uses

* FdSet init from raw fd_set

* attempt to fix borrowing, kernel_select nonblocking

* attempt continued

* fix is_set

* attempt to fix borrow checking within select

* fix in-select borrow, fix warnings

* cleanup

* fix a warning

* fix serious typo

* fix typo

* debugging

* debugging

* prints

* prints!!

* debugging

* debugging

* debugging

* debugging

* debug

* debuging

* update

* debug

* debug

* debug

* debug

* debug

* cleanup and refactor

* fixing the nfds parameter for kernel select

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* rawfd in socketdesc

* increase timeout

* change gettimeofday log level

* migrate FdSet, nodelay getsockopt, accept rawfd

* small fix

* store rawfd in acpt/lst/cnct, fix INPROGRESS

* fix ownership

* of course domain sockets don't have raw_sys_fd

* fix warnings

* fix warnings

* small coding style fixes

* add blank line

* add TCP level options

* getsockopt tcp without kernel

* add FdSet::clear back for tests

* fix warnings

* fix warning

* revert select sleep

* fix PR suggestions

* refactor kernel select logic

* reorganize

* update

---------

Co-authored-by: Nicholas Renner <nicholassrenner@gmail.com>

src/interface/comm.rs

@@ -2,7 +2,8 @@
 // //
 // //
 
-use std::mem::size_of;
+use crate::interface;
+use std::mem::{size_of};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::fs::read_to_string;
 use std::str::from_utf8;
@@ -206,7 +207,7 @@ pub struct SockaddrV6 {
 #[derive(Debug)]
 pub struct Socket {
     pub refcnt: i32,
-    raw_sys_fd: i32
+    pub raw_sys_fd: i32
 }
 
 impl Socket {
@@ -340,8 +341,8 @@ impl Socket {
 
     pub fn setsockopt(&self, level: i32, optname: i32, optval: i32) -> i32 {
         let valbuf = optval;
-        let sor =  unsafe{libc::setsockopt(self.raw_sys_fd, level, optname, (&valbuf as *const i32).cast::<libc::c_void>(), size_of::<i32>() as u32)};
-        sor
+        let ret =  unsafe{libc::setsockopt(self.raw_sys_fd, level, optname, (&valbuf as *const i32).cast::<libc::c_void>(), size_of::<i32>() as u32)};
+        ret
     }
 
     pub fn shutdown(&self, how: i32) -> i32 {
@@ -366,3 +367,95 @@ impl Drop for Socket {
 pub fn getifaddrs_from_file() -> String {
     read_to_string(NET_DEV_FILENAME).expect("No net_devices file present!").to_owned()
 }
+
+// Implementations of select related FD_SET structure
+pub struct FdSet(libc::fd_set);
+
+impl FdSet {
+    pub fn new() -> FdSet {
+        unsafe {
+            let mut raw_fd_set = std::mem::MaybeUninit::<libc::fd_set>::uninit();
+            libc::FD_ZERO(raw_fd_set.as_mut_ptr());
+            FdSet(raw_fd_set.assume_init())
+        }
+    }
+
+    pub fn new_from_ptr(raw_fdset_ptr: *const libc::fd_set) -> &'static mut FdSet {
+        unsafe {
+            &mut *(raw_fdset_ptr as *mut FdSet)
+        }
+    }
+
+    // copy the src FdSet into self
+    pub fn copy_from(&mut self, src_fds: &FdSet) {
+        unsafe {
+            std::ptr::copy_nonoverlapping(&src_fds.0 as *const libc::fd_set, &mut self.0 as *mut libc::fd_set, 1);
+        }
+    }
+
+    // turn off the fd bit in fd_set (currently only used by the tests)
+    #[allow(dead_code)]
+    pub fn clear(&mut self, fd: i32) {
+        unsafe { libc::FD_CLR(fd, &mut self.0) }
+    }
+
+    // turn on the fd bit in fd_set
+    pub fn set(&mut self, fd: i32) {
+        unsafe { libc::FD_SET(fd, &mut self.0) }
+    }
+
+    // return true if the bit for fd is set, false otherwise
+    pub fn is_set(&self, fd: i32) -> bool {
+        unsafe { libc::FD_ISSET(fd, &self.0) }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        let fd_array: &[u8] = unsafe {
+            std::slice::from_raw_parts(&self.0 as *const _ as *const u8, size_of::<libc::fd_set>())
+        };
+        fd_array.iter().all(|&byte| byte == 0)
+    }
+
+    // for each fd, if kernel_fds turned it on, then self will turn the corresponding tranlated fd on
+    pub fn set_from_kernelfds_and_translate(&mut self, kernel_fds: &FdSet, nfds: i32, rawfd_lindfd_tuples: &Vec<(i32, i32)>) {
+        for fd in 0..nfds {
+            if !kernel_fds.is_set(fd) {
+                continue;
+            }
+            // translate and set
+            if let Some((_, lindfd)) = rawfd_lindfd_tuples.iter().find(|(rawfd, _)| *rawfd == fd) {
+                self.set(*lindfd);
+            }
+        }
+    }
+}
+
+// for unwrapping in kernel_select
+fn to_fdset_ptr(opt: Option<&mut FdSet>) -> *mut libc::fd_set {
+    match opt {
+        None => std::ptr::null_mut(),
+        Some(&mut FdSet(ref mut raw_fd_set)) => raw_fd_set,
+    }
+}
+
+pub fn kernel_select(nfds: libc::c_int, readfds: Option<&mut FdSet>, writefds: Option<&mut FdSet>, errorfds: Option<&mut FdSet>) -> i32 {
+    // Call libc::select and store the result
+    let result = unsafe {
+        // Create a timeval struct with zero timeout
+
+        let mut kselect_timeout = libc::timeval {
+                tv_sec: 0,  // 0 seconds
+                tv_usec: 0, // 0 microseconds
+            };
+
+        libc::select(
+            nfds,
+            to_fdset_ptr(readfds),
+            to_fdset_ptr(writefds),
+            to_fdset_ptr(errorfds),
+            &mut kselect_timeout as *mut libc::timeval,
+        )
+    };
+
+    return result;
+}

src/interface/types.rs

@@ -192,6 +192,7 @@ pub union Arg {
   pub dispatch_constsigsett: *const SigsetType,
   pub dispatch_structitimerval: *mut ITimerVal,
   pub dispatch_conststructitimerval: *const ITimerVal,
+  pub dispatch_fdset: *mut libc::fd_set,
 }
 
 
@@ -271,6 +272,15 @@ pub fn get_mutcbuf_null(union_argument: Arg) -> Result<Option<*mut u8>, i32> {
     return Ok(None);
 }
 
+pub fn get_fdset(union_argument: Arg) -> Result<Option<&'static mut interface::FdSet>, i32> {
+    let data: *mut libc::fd_set = unsafe{union_argument.dispatch_fdset};
+    if !data.is_null() {
+        let internal_fds: &mut interface::FdSet = interface::FdSet::new_from_ptr(data);
+        return Ok(Some(internal_fds));
+    }
+    return Ok(None);
+}
+
 pub fn get_cstr<'a>(union_argument: Arg) -> Result<&'a str, i32> {
 
     //first we check that the pointer is not null 
@@ -412,49 +422,6 @@ pub fn get_sockpair<'a>(union_argument: Arg) -> Result<&'a mut SockPair, i32> {
     return Err(syscall_error(Errno::EFAULT, "dispatcher", "input data not valid"));
 }
 
-// turn on the fd bit in fd_set
-pub fn fd_set_insert(fd_set: *mut u8, fd: i32) {
-    let byte_offset = fd / 8;
-    let byte_ptr = fd_set.wrapping_offset(byte_offset as isize);
-    let bit_offset = fd & 0b111;
-    unsafe{*byte_ptr |= 1 << bit_offset;}
-}
-
-// turn off the fd bit in fd_set
-pub fn fd_set_remove(fd_set: *mut u8, fd: i32) {
-    let byte_offset = fd / 8;
-    let byte_ptr = fd_set.wrapping_offset(byte_offset as isize);
-    let bit_offset = fd & 0b111;
-    unsafe{*byte_ptr &= !(1 << bit_offset);}
-}
-
-// return true if the bit for fd is set in fd_set, false otherwise
-pub fn fd_set_check_fd(fd_set: *const u8, fd: i32) -> bool {
-    let byte_offset = fd / 8;
-    let byte_ptr = fd_set.wrapping_offset(byte_offset as isize);
-    let bit_offset = fd & 0b111;
-    return (unsafe{*byte_ptr}) & (1 << bit_offset) != 0;
-}
-
-pub fn fd_set_copy(src_set: *const u8, dst_set: *mut u8, nfds: i32) {
-    for fd in 0..nfds {
-        if interface::fd_set_check_fd(src_set, fd) {
-            interface::fd_set_insert(dst_set, fd);
-        } else {
-            interface::fd_set_remove(dst_set, fd);
-        }
-    }
-}
-
-pub fn fd_set_is_empty(fd_set: *const u8, highest_fd: i32) -> bool {
-    for fd in 0..highest_fd + 1 {
-        if fd_set_check_fd(fd_set, fd) {
-            return false;
-        }
-    }
-    return true;
-}
-
 pub fn get_sockaddr(union_argument: Arg, addrlen: u32) -> Result<interface::GenSockaddr, i32> {
     let pointer = unsafe{union_argument.dispatch_constsockaddrstruct};
     if !pointer.is_null() {    

src/safeposix/cage.rs

@@ -40,6 +40,8 @@ pub struct StreamDesc {
 #[derive(Debug, Clone)]
 pub struct SocketDesc {
     pub flags: i32,
+    pub domain: i32,
+    pub rawfd: i32,
     pub handle: interface::RustRfc<interface::RustLock<SocketHandle>>,
     pub advlock: interface::RustRfc<interface::AdvisoryLock>,
 }

src/safeposix/dispatcher.rs

@@ -386,7 +386,7 @@ pub extern "C" fn dispatcher(cageid: u64, callnum: i32, arg1: Arg, arg2: Arg, ar
             if nfds < 0 { //RLIMIT_NOFILE check as well?
                 return syscall_error(Errno::EINVAL, "select", "The number of fds passed was invalid");
             } 
-            check_and_dispatch!(cage.select_syscall, Ok::<i32, i32>(nfds), interface::get_mutcbuf_null(arg2), interface::get_mutcbuf_null(arg3), interface::get_mutcbuf_null(arg4), interface::duration_fromtimeval(arg5))
+            check_and_dispatch!(cage.select_syscall, Ok::<i32, i32>(nfds), interface::get_fdset(arg2), interface::get_fdset(arg3), interface::get_fdset(arg4), interface::duration_fromtimeval(arg5))
         }
         POLL_SYSCALL => {
             let nfds = get_onearg!(interface::get_usize(arg2));

src/safeposix/net.rs

@@ -82,7 +82,8 @@ pub struct UnixSocketInfo {
 #[derive(Debug)]
 pub struct SocketHandle {
     pub innersocket: Option<interface::Socket>,
-    pub options: i32,
+    pub socket_options: i32,
+    pub tcp_options: i32,
     pub state: ConnState,
     pub protocol: i32,
     pub domain: i32,
@@ -387,3 +388,33 @@ impl NetMetadata {
         domainsock_paths
     }
 }
+
+pub struct SelectInetInfo {
+    pub rawfd_lindfd_tuples: Vec<(i32, i32)>,
+    pub kernel_fds: interface::FdSet,
+    pub highest_raw_fd: i32,
+}
+
+impl SelectInetInfo {
+    pub fn new() -> Self {
+        SelectInetInfo {
+            rawfd_lindfd_tuples: Vec::new(),
+            kernel_fds: interface::FdSet::new(),
+            highest_raw_fd: 0,
+        }
+    }
+}
+
+pub fn update_readfds_from_kernel_select(readfds: &mut interface::FdSet, inet_info: &mut SelectInetInfo, retval: &mut i32) -> i32 {
+    let kernel_ret;
+    // note that this select call always have timeout = 0, so it doesn't block
+
+    kernel_ret = interface::kernel_select(inet_info.highest_raw_fd + 1, Some(&mut inet_info.kernel_fds), None, None);
+    if kernel_ret > 0 {
+        // increment retval of our select
+        *retval += kernel_ret;
+        // translate the kernel checked fds to lindfds, and add to our new_writefds
+        readfds.set_from_kernelfds_and_translate(&mut inet_info.kernel_fds, inet_info.highest_raw_fd + 1, &inet_info.rawfd_lindfd_tuples);
+    }
+    return kernel_ret;
+}