From 981de380e237ab402a85075a29fe698cacaff9ae Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 23 Jun 2023 20:06:09 +0800 Subject: [PATCH 1/5] Migrate to WSAWaitForMultipleEvents on Windows --- include/n2n_typedefs.h | 3 + src/edge.c | 17 +++- src/edge_utils.c | 174 ++++++++++++++++++++++++++++++----------- 3 files changed, 146 insertions(+), 48 deletions(-) diff --git a/include/n2n_typedefs.h b/include/n2n_typedefs.h index 582b59084..ad82db0bf 100644 --- a/include/n2n_typedefs.h +++ b/include/n2n_typedefs.h @@ -705,6 +705,9 @@ struct n2n_edge { /* Status */ bool *keep_running; /**< Pointer to edge loop stop/go flag */ +#ifdef WIN32 + HANDLE *stop_event_handle; /**< Pointer to edge loop stop signal */ +#endif struct peer_info *curr_sn; /**< Currently active supernode. */ uint8_t sn_wait; /**< Whether we are waiting for a supernode response. */ uint8_t sn_pong; /**< Whether we have seen a PONG since last time reset. */ diff --git a/src/edge.c b/src/edge.c index d2c028877..302165197 100644 --- a/src/edge.c +++ b/src/edge.c @@ -43,7 +43,7 @@ #include "uthash.h" // for UT_hash_handle, HASH_ADD, HASH_C... #ifdef WIN32 -#include +#include #include #else #include // for inet_addr, inet_ntop @@ -960,6 +960,10 @@ static void daemonize () { /* *************************************************** */ static bool keep_on_running = true; +#ifdef WIN32 +static HANDLE stop_event_handle; +static HANDLE main_thread; +#endif #if defined(__linux__) || defined(WIN32) #ifdef WIN32 @@ -980,6 +984,12 @@ BOOL WINAPI term_handler(DWORD sig) keep_on_running = false; #ifdef WIN32 + if (!SetEvent(stop_event_handle)) { + traceEvent(TRACE_ERROR, "failed to set stop signal, you may experience slow shutdown, error code: %d", GetLastError()); + } + // if (!CancelSynchronousIo(main_thread)) { + // traceEvent(TRACE_ERROR, "failed to cancel synchronous io, you may experience slow shutdown, error code: %d", GetLastError()); + // } switch (sig) { case CTRL_CLOSE_EVENT: case CTRL_LOGOFF_EVENT: @@ -1341,7 +1351,10 @@ int main (int argc, char* argv[]) { signal(SIGINT, term_handler); #endif #ifdef WIN32 - SetConsoleCtrlHandler(term_handler, TRUE); + SetConsoleCtrlHandler(term_handler, true); + main_thread = GetCurrentThread(); + stop_event_handle = CreateEvent(NULL, true, false, NULL); + eee->stop_event_handle = stop_event_handle; #endif eee->keep_running = &keep_on_running; diff --git a/src/edge_utils.c b/src/edge_utils.c index 8d6568ae7..5dac18873 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -41,7 +41,7 @@ #include "uthash.h" // for UT_hash_handle, HASH_COUNT, HASH... #ifdef WIN32 -#include +#include #include #include "edge_utils_win32.h" #else @@ -2873,6 +2873,17 @@ void print_edge_stats (const n2n_edge_t *eee) { /* ************************************** */ +bool add_read_event_select(int *total_events, HANDLE *events, SOCKET socket) { + HANDLE event_handle = WSACreateEvent(); + int result = WSAEventSelect(socket, event_handle, FD_READ); + if (result != NO_ERROR) { + traceEvent(TRACE_ERROR, "can't select event, error code: %d", WSAGetLastError()); + return false; + } + events[*total_events] = event_handle; + *total_events += 1; + return true; +} int run_edge_loop (n2n_edge_t *eee) { @@ -2895,7 +2906,6 @@ int run_edge_loop (n2n_edge_t *eee) { HANDLE tun_read_thread = startTunReadThread(&arg); #endif - *eee->keep_running = true; update_supernode_reg(eee, time(NULL)); /* Main loop @@ -2907,10 +2917,73 @@ int run_edge_loop (n2n_edge_t *eee) { while(*eee->keep_running) { - int rc, max_sock = 0; + bool management_has_message = false; + bool sock_has_message = false; + bool multicast_has_message = false; +#ifndef WIN32 + bool tuntap_has_message = false; +#endif + +#ifdef WIN32 + int total_events = 0; + HANDLE events[WSA_MAXIMUM_WAIT_EVENTS] = {0}; + bool *event_message_map[WSA_MAXIMUM_WAIT_EVENTS] = {0}; + events[total_events] = eee->stop_event_handle; + total_events += 1; + event_message_map[total_events] = &management_has_message; + if (!add_read_event_select(&total_events, events, eee->udp_mgmt_sock)) { + break; + } + if (eee->sock >= 0) { + event_message_map[total_events] = &sock_has_message; + if (!add_read_event_select(&total_events, events, eee->sock)) { + break; + } + } +#ifndef SKIP_MULTICAST_PEERS_DISCOVERY + if (eee->conf.allow_p2p && eee->conf.preferred_sock.family == AF_INVALID) { + event_message_map[total_events] = &multicast_has_message; + if (!add_read_event_select(&total_events, events, eee->udp_multicast_sock)) { + break; + } + } +#endif + DWORD wait_time_ms = (eee->sn_wait ? SOCKET_TIMEOUT_INTERVAL_SECS / 10 + 1 : SOCKET_TIMEOUT_INTERVAL_SECS) * 1000; + DWORD result = WSAWaitForMultipleEvents(total_events, events, false, wait_time_ms, true); + for (int i = 0; i < total_events; i++) { + HANDLE current_event = events[i]; + if (current_event == NULL) { + break; + } + if (current_event != eee->stop_event_handle) { + if (!WSACloseEvent(current_event)) { + traceEvent(TRACE_ERROR, "can't close event, error code: %d", WSAGetLastError()); + break; + } + } + } + + traceEvent(TRACE_NORMAL, "wait for multiple events result: %d", result); + if (result == WSA_WAIT_IO_COMPLETION) { + traceEvent(TRACE_NORMAL, "WSA_WAIT_IO_COMPLETION"); + } + if (result == WSA_WAIT_FAILED) { + traceEvent(TRACE_ERROR, "can't wait for events, error code: %d", WSAGetLastError()); + break; + } + if (result >= WSA_WAIT_EVENT_0 && result <= WSA_WAIT_EVENT_0 + total_events - 1) { + int index = result - WSA_WAIT_EVENT_0; + HANDLE signaled_event = events[index]; + traceEvent(TRACE_NORMAL, "signal event: %d, stop event: %d", signaled_event, eee->stop_event_handle); + if (signaled_event == eee->stop_event_handle) { + traceEvent(TRACE_NORMAL, "exit signal received"); + break; + } + *event_message_map[index] = true; + } +#else + int max_sock = 0; fd_set socket_mask; - struct timeval wait_time; - time_t now; FD_ZERO(&socket_mask); @@ -2933,11 +3006,20 @@ int run_edge_loop (n2n_edge_t *eee) { FD_SET(eee->device.fd, &socket_mask); max_sock = max(max_sock, eee->device.fd); #endif - + struct timeval wait_time; wait_time.tv_sec = (eee->sn_wait) ? (SOCKET_TIMEOUT_INTERVAL_SECS / 10 + 1) : (SOCKET_TIMEOUT_INTERVAL_SECS); wait_time.tv_usec = 0; - rc = select(max_sock + 1, &socket_mask, NULL, NULL, &wait_time); - now = time(NULL); + traceEvent(TRACE_NORMAL, "start waiting for select"); + int rc = select(max_sock + 1, &socket_mask, NULL, NULL, &wait_time); + traceEvent(TRACE_NORMAL, "finish waiting for select"); + + management_has_message = FD_ISSET(eee->udp_mgmt_sock, &socket_mask); + sock_has_message = FD_ISSET(eee->sock, &socket_mask); + multicast_has_message = FD_ISSET(eee->udp_multicast_sock, &socket_mask); + tuntap_has_message = FD_ISSET(eee->device.fd, &socket_mask); +#endif + + time_t now = time(NULL); // make sure ciphers are updated before the packet is treated if((now - lastTransop) > TRANSOP_TICK_INTERVAL) { @@ -2946,56 +3028,54 @@ int run_edge_loop (n2n_edge_t *eee) { eee->transop.tick(&eee->transop, now); } - if(rc > 0) { - // any or all of the FDs could have input; check them all + // any or all of the FDs could have input; check them all - // external - if((eee->sock >= 0) && FD_ISSET(eee->sock, &socket_mask)) { - if(0 != fetch_and_eventually_process_data(eee, eee->sock, - pktbuf, &expected, &position, - now)) { - *eee->keep_running = false; - break; - } - if(eee->conf.connect_tcp) { - if((expected >= N2N_PKT_BUF_SIZE) || (position >= N2N_PKT_BUF_SIZE)) { - // something went wrong, possibly even before - // e.g. connection failure/closure in the middle of transmission (between len & data) - supernode_disconnect(eee); - eee->sn_wait = 1; - - expected = sizeof(uint16_t); - position = 0; - } + // external + if (sock_has_message) { + if(0 != fetch_and_eventually_process_data(eee, eee->sock, + pktbuf, &expected, &position, + now)) { + *eee->keep_running = false; + break; + } + if(eee->conf.connect_tcp) { + if((expected >= N2N_PKT_BUF_SIZE) || (position >= N2N_PKT_BUF_SIZE)) { + // something went wrong, possibly even before + // e.g. connection failure/closure in the middle of transmission (between len & data) + supernode_disconnect(eee); + eee->sn_wait = 1; + + expected = sizeof(uint16_t); + position = 0; } } + } #ifndef SKIP_MULTICAST_PEERS_DISCOVERY - if(FD_ISSET(eee->udp_multicast_sock, &socket_mask)) { - if(0 != fetch_and_eventually_process_data(eee, eee->udp_multicast_sock, - pktbuf, &expected, &position, - now)) { - *eee->keep_running = false; - break; - } + if (multicast_has_message) { + if(0 != fetch_and_eventually_process_data(eee, eee->udp_multicast_sock, + pktbuf, &expected, &position, + now)) { + *eee->keep_running = false; + break; } + } #endif - if(FD_ISSET(eee->udp_mgmt_sock, &socket_mask)) { - // read from the management port socket - readFromMgmtSocket(eee); + if (management_has_message) { + // read from the management port socket + readFromMgmtSocket(eee); - if(!(*eee->keep_running)) - break; - } + if(!(*eee->keep_running)) + break; + } #ifndef WIN32 - if(FD_ISSET(eee->device.fd, &socket_mask)) { - // read an ethernet frame from the TAP socket; write on the IP socket - edge_read_from_tap(eee); - } -#endif + if (tuntap_has_message) { + // read an ethernet frame from the TAP socket; write on the IP socket + edge_read_from_tap(eee); } +#endif // finished processing select data update_supernode_reg(eee, now); @@ -3056,7 +3136,9 @@ int run_edge_loop (n2n_edge_t *eee) { send_unregister_super(eee); #ifdef WIN32 + traceEvent(TRACE_NORMAL, "start waiting for tun read thread"); WaitForSingleObject(tun_read_thread, INFINITE); + traceEvent(TRACE_NORMAL, "finish waiting for tun read thread"); #endif supernode_disconnect(eee); From 888a43d50dd0f8ea564151c6d7409459aa4b8fe1 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 23 Jun 2023 20:23:57 +0800 Subject: [PATCH 2/5] Remove testing CancelSynchronousIo code --- src/edge.c | 5 ----- src/edge_utils.c | 6 +----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/edge.c b/src/edge.c index 302165197..e4bf2a4a7 100644 --- a/src/edge.c +++ b/src/edge.c @@ -962,7 +962,6 @@ static void daemonize () { static bool keep_on_running = true; #ifdef WIN32 static HANDLE stop_event_handle; -static HANDLE main_thread; #endif #if defined(__linux__) || defined(WIN32) @@ -987,9 +986,6 @@ BOOL WINAPI term_handler(DWORD sig) if (!SetEvent(stop_event_handle)) { traceEvent(TRACE_ERROR, "failed to set stop signal, you may experience slow shutdown, error code: %d", GetLastError()); } - // if (!CancelSynchronousIo(main_thread)) { - // traceEvent(TRACE_ERROR, "failed to cancel synchronous io, you may experience slow shutdown, error code: %d", GetLastError()); - // } switch (sig) { case CTRL_CLOSE_EVENT: case CTRL_LOGOFF_EVENT: @@ -1352,7 +1348,6 @@ int main (int argc, char* argv[]) { #endif #ifdef WIN32 SetConsoleCtrlHandler(term_handler, true); - main_thread = GetCurrentThread(); stop_event_handle = CreateEvent(NULL, true, false, NULL); eee->stop_event_handle = stop_event_handle; #endif diff --git a/src/edge_utils.c b/src/edge_utils.c index 5dac18873..db387dbcb 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -2949,7 +2949,7 @@ int run_edge_loop (n2n_edge_t *eee) { } #endif DWORD wait_time_ms = (eee->sn_wait ? SOCKET_TIMEOUT_INTERVAL_SECS / 10 + 1 : SOCKET_TIMEOUT_INTERVAL_SECS) * 1000; - DWORD result = WSAWaitForMultipleEvents(total_events, events, false, wait_time_ms, true); + DWORD result = WSAWaitForMultipleEvents(total_events, events, false, wait_time_ms, false); for (int i = 0; i < total_events; i++) { HANDLE current_event = events[i]; if (current_event == NULL) { @@ -2964,9 +2964,6 @@ int run_edge_loop (n2n_edge_t *eee) { } traceEvent(TRACE_NORMAL, "wait for multiple events result: %d", result); - if (result == WSA_WAIT_IO_COMPLETION) { - traceEvent(TRACE_NORMAL, "WSA_WAIT_IO_COMPLETION"); - } if (result == WSA_WAIT_FAILED) { traceEvent(TRACE_ERROR, "can't wait for events, error code: %d", WSAGetLastError()); break; @@ -2974,7 +2971,6 @@ int run_edge_loop (n2n_edge_t *eee) { if (result >= WSA_WAIT_EVENT_0 && result <= WSA_WAIT_EVENT_0 + total_events - 1) { int index = result - WSA_WAIT_EVENT_0; HANDLE signaled_event = events[index]; - traceEvent(TRACE_NORMAL, "signal event: %d, stop event: %d", signaled_event, eee->stop_event_handle); if (signaled_event == eee->stop_event_handle) { traceEvent(TRACE_NORMAL, "exit signal received"); break; From 0fc8f950ad394f60e451ccb8526508bff37c9654 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 23 Jun 2023 20:26:50 +0800 Subject: [PATCH 3/5] Change all winsock includes to winsock2 --- src/edge_management.c | 2 +- src/example_sn_embed.c | 2 +- src/management.h | 2 +- src/n2n.c | 2 +- src/network_traffic_filter.c | 2 +- src/sn_management.c | 2 +- src/sn_utils.c | 2 +- src/supernode.c | 2 +- src/wire.c | 2 +- tools/n2n-portfwd.c | 2 +- tools/n2n-route.c | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/edge_management.c b/src/edge_management.c index 53e0b6feb..7381f1c3b 100644 --- a/src/edge_management.c +++ b/src/edge_management.c @@ -36,7 +36,7 @@ #include "uthash.h" // for UT_hash_handle, HASH_ITER #ifdef WIN32 -#include +#include #include #include "edge_utils_win32.h" #else diff --git a/src/example_sn_embed.c b/src/example_sn_embed.c index 713d068f4..cdd16ed2d 100644 --- a/src/example_sn_embed.c +++ b/src/example_sn_embed.c @@ -22,7 +22,7 @@ #include "n2n.h" // for n2n_sn_t, open_socket, run_sn_loop, sn_init #ifdef WIN32 -#include +#include #else #include // for INADDR_ANY, INADDR_LOOPBACK #endif diff --git a/src/management.h b/src/management.h index 3e954580d..53592bdbf 100644 --- a/src/management.h +++ b/src/management.h @@ -17,7 +17,7 @@ #include "strbuf.h" #ifdef WIN32 -#include +#include #else #include // for sockaddr, sockaddr_storage, socklen_t #endif diff --git a/src/n2n.c b/src/n2n.c index c5a65adf5..335ca1175 100644 --- a/src/n2n.c +++ b/src/n2n.c @@ -31,7 +31,7 @@ #include "uthash.h" // for UT_hash_handle, HASH_DEL, HASH_ITER, HAS... #ifdef WIN32 -#include +#include #include #include #else diff --git a/src/network_traffic_filter.c b/src/network_traffic_filter.c index fc8b4c14a..4ba0d0bdc 100644 --- a/src/network_traffic_filter.c +++ b/src/network_traffic_filter.c @@ -26,7 +26,7 @@ #include "uthash.h" // for UT_hash_handle, HASH_ITER, HASH_DEL #ifdef WIN32 -#include +#include #include #else #include // for inet_ntoa, inet_addr diff --git a/src/sn_management.c b/src/sn_management.c index 3dae672fa..4954cdfa8 100644 --- a/src/sn_management.c +++ b/src/sn_management.c @@ -36,7 +36,7 @@ #include "uthash.h" // for UT_hash_handle, HASH_ITER, HASH_COUNT #ifdef WIN32 -#include +#include #include "edge_utils_win32.h" #else #include // for sendto, socklen_t diff --git a/src/sn_utils.c b/src/sn_utils.c index 9b57f5872..88686f7e7 100644 --- a/src/sn_utils.c +++ b/src/sn_utils.c @@ -41,7 +41,7 @@ #include "uthash.h" // for UT_hash_handle, HASH_ITER, HASH_DEL #ifdef WIN32 -#include +#include #include #else #include // for inet_addr, inet_ntoa diff --git a/src/supernode.c b/src/supernode.c index 1c640f1cf..2f604162a 100644 --- a/src/supernode.c +++ b/src/supernode.c @@ -36,7 +36,7 @@ #include "uthash.h" // for UT_hash_handle, HASH_ITER, HASH_ADD_STR #ifdef WIN32 -#include +#include #include #else #include // for inet_addr diff --git a/src/wire.c b/src/wire.c index af1011fbc..765a8e7b2 100644 --- a/src/wire.c +++ b/src/wire.c @@ -34,7 +34,7 @@ #include "n2n_wire.h" // for decode_PACKET, decode_PEER_INFO, decode_QUER... #ifdef WIN32 -#include +#include #include #else #include // for sockaddr_in, sockaddr_in6, in6_addr, in_addr diff --git a/tools/n2n-portfwd.c b/tools/n2n-portfwd.c index 89dabcc43..f01fb1070 100644 --- a/tools/n2n-portfwd.c +++ b/tools/n2n-portfwd.c @@ -34,7 +34,7 @@ #include "random_numbers.h" // for n2n_rand, n2n_seed, n2n_srand #ifdef WIN32 -#include +#include #include #else #include // for sockaddr_in, htonl, htons, INADDR_LOOP... diff --git a/tools/n2n-route.c b/tools/n2n-route.c index a7d3cddcb..6ea01f189 100644 --- a/tools/n2n-route.c +++ b/tools/n2n-route.c @@ -39,7 +39,7 @@ #endif #ifdef WIN32 -#include +#include #include #else #include // for inet_pton From 5b51d90bcc131e9a070973049848f730e0c4d858 Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 23 Jun 2023 20:32:55 +0800 Subject: [PATCH 4/5] Fix missing ifdef WIN32 on add_read_event_select --- src/edge_utils.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/edge_utils.c b/src/edge_utils.c index db387dbcb..0a880e0d6 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -2873,6 +2873,7 @@ void print_edge_stats (const n2n_edge_t *eee) { /* ************************************** */ +#ifdef WIN32 bool add_read_event_select(int *total_events, HANDLE *events, SOCKET socket) { HANDLE event_handle = WSACreateEvent(); int result = WSAEventSelect(socket, event_handle, FD_READ); @@ -2884,6 +2885,7 @@ bool add_read_event_select(int *total_events, HANDLE *events, SOCKET socket) { *total_events += 1; return true; } +#endif int run_edge_loop (n2n_edge_t *eee) { From 1ca14fc68fa1f4371b94f491027813c315a7394d Mon Sep 17 00:00:00 2001 From: Tony Date: Fri, 22 Dec 2023 23:26:01 +0800 Subject: [PATCH 5/5] Change WIN32 to _WIN32 --- include/n2n_typedefs.h | 2 +- src/edge_utils.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/n2n_typedefs.h b/include/n2n_typedefs.h index 971ac4366..89032ae4f 100644 --- a/include/n2n_typedefs.h +++ b/include/n2n_typedefs.h @@ -706,7 +706,7 @@ struct n2n_edge { /* Status */ bool *keep_running; /**< Pointer to edge loop stop/go flag */ -#ifdef WIN32 +#ifdef _WIN32 HANDLE *stop_event_handle; /**< Pointer to edge loop stop signal */ #endif struct peer_info *curr_sn; /**< Currently active supernode. */ diff --git a/src/edge_utils.c b/src/edge_utils.c index a187302c6..5688f42d5 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -2872,7 +2872,7 @@ void print_edge_stats (const n2n_edge_t *eee) { /* ************************************** */ -#ifdef WIN32 +#ifdef _WIN32 bool add_read_event_select(int *total_events, HANDLE *events, SOCKET socket) { HANDLE event_handle = WSACreateEvent(); int result = WSAEventSelect(socket, event_handle, FD_READ); @@ -2921,11 +2921,11 @@ int run_edge_loop (n2n_edge_t *eee) { bool management_has_message = false; bool sock_has_message = false; bool multicast_has_message = false; -#ifndef WIN32 +#ifndef _WIN32 bool tuntap_has_message = false; #endif -#ifdef WIN32 +#ifdef _WIN32 int total_events = 0; HANDLE events[WSA_MAXIMUM_WAIT_EVENTS] = {0}; bool *event_message_map[WSA_MAXIMUM_WAIT_EVENTS] = {0};