diff --git a/Common/Cpp/Sockets/ServerSocket_POSIX.cpp b/Common/Cpp/Sockets/ServerSocket_POSIX.cpp index d24d8c8f50..47736e751b 100644 --- a/Common/Cpp/Sockets/ServerSocket_POSIX.cpp +++ b/Common/Cpp/Sockets/ServerSocket_POSIX.cpp @@ -30,30 +30,38 @@ ServerSocket_POSIX::ServerSocket_POSIX(ThreadPool& thread_pool) } void ServerSocket_POSIX::start(const std::string& address, uint16_t port){ + uint64_t generation = 0; { std::lock_guard lg(m_lock); - if (m_state.load(std::memory_order_relaxed) != State::NOT_RUNNING - || m_starting){ + if (m_starting || + m_state.load(std::memory_order_relaxed) != State::NOT_RUNNING + ){ return; } m_starting = true; + m_generation++; + generation = m_generation; } // Use the plain dispatch() rather than dispatch_now_blocking(): // start() may be called from a thread that is itself running on // this pool, and blocking here until a pool thread is free could - // deadlock a capped pool. The returned AsyncTask still lets - // close() join the listener thread. + // deadlock a capped pool. try{ AsyncTask task = m_thread_pool.dispatch([=, this]{ - accept_thread_loop(address, port); + accept_thread_loop(address, port, generation); }); { - // m_task_lock, not m_lock: close() only reads the slot - // after it has waited for m_starting to clear, so this - // assignment is the only writer besides close(). + // Append, never replace: replacing the slot would destroy + // the previous task's AsyncTask, and its destructor blocks + // until that task has finished. A task from an earlier + // start() may still be running when this start() runs (the + // caller's wait timed out while the pool was starved). It + // notices its stale generation when it gets scheduled and + // exits without listening, so it only has to outlive this + // call; close() joins the whole list. std::lock_guard lg(m_task_lock); - m_accept_thread = std::move(task); + m_accept_threads.push_back(std::move(task)); } }catch (...){ { @@ -61,7 +69,10 @@ void ServerSocket_POSIX::start(const std::string& address, uint16_t port){ m_starting = false; m_cv.notify_all(); } - m_listeners.run_method(&Listener::on_listen_finished, "Failed to start the listener thread."); + m_listeners.run_method( + &Listener::on_listen_finished, + "Failed to start the listener thread." + ); throw; } @@ -119,23 +130,24 @@ void ServerSocket_POSIX::close() noexcept{ return; } - AsyncTask accept_thread; AsyncTask client_thread; + std::vector accept_threads; { // m_task_lock, not m_lock: moving the tasks out destroys the // old AsyncTask objects, and their destructors wait for the // tasks to finish - which they can only do if they can acquire // m_lock. std::lock_guard lg(m_task_lock); - accept_thread = std::move(m_accept_thread); - m_accept_thread = AsyncTask(); + accept_threads.swap(m_accept_threads); client_thread = std::move(m_client_thread); m_client_thread = AsyncTask(); } // The threads notice the new state within a few milliseconds and // close their own sockets. - accept_thread.wait_and_ignore_exceptions(); + for (auto& task : accept_threads){ + task.wait_and_ignore_exceptions(); + } client_thread.wait_and_ignore_exceptions(); { @@ -151,8 +163,9 @@ size_t ServerSocket_POSIX::send(const void* data, size_t bytes, std::chrono::mil int socket; { std::lock_guard lg(m_lock); - socket = - (m_state.load(std::memory_order_relaxed) == State::CONNECTED) ? m_client : -1; + socket = m_state.load(std::memory_order_relaxed) == State::CONNECTED + ? m_client + : -1; } if (socket == -1){ return 0; @@ -209,7 +222,18 @@ size_t ServerSocket_POSIX::send(const void* data, size_t bytes, std::chrono::mil return sent; } -void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port){ +void ServerSocket_POSIX::report_listen_finished(uint64_t generation, const std::string& error){ + bool current; + { + std::lock_guard lg(m_lock); + current = (m_generation == generation); + } + if (current){ + m_listeners.run_method(&Listener::on_listen_finished, error); + } +} + +void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port, uint64_t generation){ int listen_socket = ::socket(AF_INET, SOCK_STREAM, 0); if (listen_socket == -1){ m_listeners.run_method( @@ -231,8 +255,8 @@ void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port){ if (bind(listen_socket, (struct sockaddr*)&server, sizeof(server)) == -1){ int error = errno; ::close(listen_socket); - m_listeners.run_method( - &Listener::on_listen_finished, + report_listen_finished( + generation, "Failed to bind the socket: " + std::string(strerror(error)) ); return; @@ -241,19 +265,23 @@ void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port){ if (::listen(listen_socket, 16) == -1){ int error = errno; ::close(listen_socket); - m_listeners.run_method( - &Listener::on_listen_finished, + report_listen_finished( + generation, "Failed to listen on the socket: " + std::string(strerror(error)) ); return; } std::string start_error; + bool superseded = false; { std::lock_guard lg(m_lock); - if (m_state.load(std::memory_order_relaxed) == State::DESTRUCTING){ - // close() won the race. - start_error = "The server was closed before it could start listening."; + superseded = (m_generation != generation); + if (superseded || + m_state.load(std::memory_order_relaxed) == State::DESTRUCTING + ){ + // A newer start() or close() won the race. + start_error = "The server was replaced or closed before it could start listening."; }else{ m_listen = listen_socket; listen_socket = -1; @@ -264,6 +292,14 @@ void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port){ ::close(listen_socket); } + if (superseded){ + // The attempt that started this task gave up on it while it was + // still starting up (its wait timed out). A newer attempt owns + // the listeners now; reporting this result would only confuse + // that attempt, so exit without touching them. + return; + } + m_listeners.run_method(&Listener::on_listen_finished, start_error); if (!start_error.empty()){ return; @@ -277,7 +313,10 @@ void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port){ { std::lock_guard lg(m_lock); State state = m_state.load(std::memory_order_relaxed); - if (state == State::DESTRUCTING || state == State::NOT_RUNNING){ + if (state == State::DESTRUCTING || + state == State::NOT_RUNNING || + m_generation != generation + ){ break; } listen = m_listen; diff --git a/Common/Cpp/Sockets/ServerSocket_POSIX.h b/Common/Cpp/Sockets/ServerSocket_POSIX.h index dcd624bc10..00f0549cd6 100644 --- a/Common/Cpp/Sockets/ServerSocket_POSIX.h +++ b/Common/Cpp/Sockets/ServerSocket_POSIX.h @@ -8,6 +8,7 @@ #define PokemonAutomation_ServerSocket_POSIX_H #include +#include #include #include #include "Common/Cpp/Concurrency/AsyncTask.h" @@ -52,9 +53,15 @@ class ServerSocket_POSIX final : public AbstractServerSocket{ private: - void accept_thread_loop(std::string address, uint16_t port); + void accept_thread_loop(std::string address, uint16_t port, uint64_t generation); void connection_thread_loop(); + // "on_listen_finished" callback, but only for a task that still owns + // the server. A task whose generation is stale (a newer start() ran + // while it was still starting up) must stay silent, or its result + // would corrupt the attempt that replaced it. + void report_listen_finished(uint64_t generation, const std::string& error); + private: ThreadPool& m_thread_pool; @@ -63,17 +70,28 @@ class ServerSocket_POSIX final : public AbstractServerSocket{ bool m_starting = false; int m_closing = 0; - // Protects m_accept_thread and m_client_thread. Kept separate from - // m_lock: assigning to a task slot destroys the previous task's - // AsyncTask, and its destructor waits for that task to finish - which - // requires m_lock, so the two locks must never be held at once. + // Increments on every start(). The accept task captures its own + // generation so that a task left behind by a timed-out attempt can + // tell it was replaced when it finally gets scheduled. + uint64_t m_generation = 0; + + // Protects m_accept_threads and m_client_thread. Kept separate from + // m_lock because the task's lifetime interacts with it: joining a + // task waits for it to exit, and exiting requires m_lock. Holding + // m_lock while a task slot is destroyed would deadlock, so slot + // operations happen under m_task_lock with m_lock released. Mutex m_task_lock; int m_listen = -1; int m_client = -1; bool m_close_requested = false; - AsyncTask m_accept_thread; + // One task per start() call. start() must never destroy the previous + // task's AsyncTask here: its destructor blocks until the task has + // finished, and a task from an earlier attempt may still be running + // (the caller's wait timed out while the pool was starved). close() + // joins the whole list. + std::vector m_accept_threads; AsyncTask m_client_thread; }; diff --git a/Common/Cpp/Sockets/ServerSocket_Qt.cpp b/Common/Cpp/Sockets/ServerSocket_Qt.cpp index bea653790c..ff3ac18120 100644 --- a/Common/Cpp/Sockets/ServerSocket_Qt.cpp +++ b/Common/Cpp/Sockets/ServerSocket_Qt.cpp @@ -23,7 +23,16 @@ void ServerSocket_Qt::start(const std::string& address, uint16_t port){ (m_state.load(std::memory_order_relaxed) == State::NOT_RUNNING) && m_server == nullptr && !m_starting; - m_starting = true; + // Only claim the "starting" flag when we actually start: a + // rejected call (server already running or already started) must + // not leave "m_starting" true - "close()" waits on + // "!m_starting", so a stale flag would deadlock the destructor. + // (This happens when a test's start-retry loop calls start() on + // a server whose first listen failed: "m_server" is set even on + // a failed listen, so the retry is rejected here.) + if (starting){ + m_starting = true; + } } if (!starting){ return; @@ -48,12 +57,15 @@ void ServerSocket_Qt::start(const std::string& address, uint16_t port){ m_listeners.run_method(&Listener::on_listen_finished, "Failed to start the listener."); throw; } - bool listening; + bool listening = false; + bool destructing = false; { std::lock_guard lg(m_lock); m_starting = false; + destructing = + (m_state.load(std::memory_order_relaxed) == State::DESTRUCTING); m_server = server; - if (m_state.load(std::memory_order_relaxed) == State::DESTRUCTING){ + if (destructing){ // "close()" is already in progress: it will destroy the // server we just created. Leave the state for it. }else{ @@ -66,6 +78,19 @@ void ServerSocket_Qt::start(const std::string& address, uint16_t port){ m_cv.notify_all(); } + if (!destructing && !listening){ + // The listen failed. Destroy the server now, leaving the object + // exactly as it was before this start() (no "m_server", state + // NOT_RUNNING), so the caller's start-retry loop can immediately + // try another port. The other backends behave the same: a failed + // listen leaves no listener behind. If "close()" raced us, it has + // already claimed "m_server", and the pool's lookup makes the + // second destruction a no-op. + GlobalThreadPools::qt_event_threadpool().remove_object(server); + std::lock_guard lg(m_lock); + m_server = nullptr; + } + m_listeners.run_method(&Listener::on_listen_finished, m_listen_error); } diff --git a/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp b/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp index 8f1b1d4d24..b269dc69a7 100644 --- a/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp +++ b/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp @@ -41,30 +41,38 @@ void ServerSocket_WinSocket::start(const std::string& address, uint16_t port){ // run). ensure_wsa_started(); + uint64_t generation = 0; { std::lock_guard lg(m_lock); - if (m_state.load(std::memory_order_relaxed) != State::NOT_RUNNING - || m_starting){ + if (m_starting || + m_state.load(std::memory_order_relaxed) != State::NOT_RUNNING + ){ return; } m_starting = true; + m_generation++; + generation = m_generation; } // Use the plain dispatch() rather than dispatch_now_blocking(): // start() may be called from a thread that is itself running on // this pool, and blocking here until a pool thread is free could - // deadlock a capped pool. The returned AsyncTask still lets - // close() join the listener thread. + // deadlock a capped pool. try{ AsyncTask task = m_thread_pool.dispatch([=, this]{ - accept_thread_loop(address, port); + accept_thread_loop(address, port, generation); }); { - // m_task_lock, not m_lock: close() only reads the slot - // after it has waited for m_starting to clear, so this - // assignment is the only writer besides close(). + // Append, never replace: replacing the slot would destroy + // the previous task's AsyncTask, and its destructor blocks + // until that task has finished. A task from an earlier + // start() may still be running when this start() runs (the + // caller's wait timed out while the pool was starved). It + // notices its stale generation when it gets scheduled and + // exits without listening, so it only has to outlive this + // call; close() joins the whole list. std::lock_guard lg(m_task_lock); - m_accept_thread = std::move(task); + m_accept_threads.push_back(std::move(task)); } }catch (...){ { @@ -72,7 +80,10 @@ void ServerSocket_WinSocket::start(const std::string& address, uint16_t port){ m_starting = false; m_cv.notify_all(); } - m_listeners.run_method(&Listener::on_listen_finished, "Failed to start the listener thread."); + m_listeners.run_method( + &Listener::on_listen_finished, + "Failed to start the listener thread." + ); throw; } @@ -130,7 +141,7 @@ void ServerSocket_WinSocket::close() noexcept{ return; } - AsyncTask accept_thread; + std::vector accept_threads; AsyncTask client_thread; { // m_task_lock, not m_lock: moving the tasks out destroys the @@ -138,15 +149,16 @@ void ServerSocket_WinSocket::close() noexcept{ // tasks to finish - which they can only do if they can acquire // m_lock. std::lock_guard lg(m_task_lock); - accept_thread = std::move(m_accept_thread); - m_accept_thread = AsyncTask(); + accept_threads.swap(m_accept_threads); client_thread = std::move(m_client_thread); m_client_thread = AsyncTask(); } // The threads notice the new state within a few milliseconds and // close their own sockets. - accept_thread.wait_and_ignore_exceptions(); + for (auto& task : accept_threads){ + task.wait_and_ignore_exceptions(); + } client_thread.wait_and_ignore_exceptions(); { @@ -162,8 +174,9 @@ size_t ServerSocket_WinSocket::send(const void* data, size_t bytes, std::chrono: SOCKET socket; { std::lock_guard lg(m_lock); - socket = - (m_state.load(std::memory_order_relaxed) == State::CONNECTED) ? m_client : INVALID_SOCKET; + socket = m_state.load(std::memory_order_relaxed) == State::CONNECTED + ? m_client + : INVALID_SOCKET; } if (socket == INVALID_SOCKET){ return 0; @@ -219,11 +232,22 @@ size_t ServerSocket_WinSocket::send(const void* data, size_t bytes, std::chrono: return sent; } -void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t port){ +void ServerSocket_WinSocket::report_listen_finished(uint64_t generation, const std::string& error){ + bool current; + { + std::lock_guard lg(m_lock); + current = (m_generation == generation); + } + if (current){ + m_listeners.run_method(&Listener::on_listen_finished, error); + } +} + +void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t port, uint64_t generation){ SOCKET listen_socket = ::socket(AF_INET, SOCK_STREAM, 0); if (listen_socket == INVALID_SOCKET){ - m_listeners.run_method( - &Listener::on_listen_finished, + report_listen_finished( + generation, "socket() failed: WSA Error Code: " + std::to_string(WSAGetLastError()) ); return; @@ -244,8 +268,8 @@ void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t po if (bind(listen_socket, (struct sockaddr*)&server, sizeof(server)) == SOCKET_ERROR){ int error = WSAGetLastError(); ::closesocket(listen_socket); - m_listeners.run_method( - &Listener::on_listen_finished, + report_listen_finished( + generation, "bind() failed: WSA Error Code: " + std::to_string(error) ); return; @@ -254,19 +278,23 @@ void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t po if (::listen(listen_socket, 16) == SOCKET_ERROR){ int error = WSAGetLastError(); ::closesocket(listen_socket); - m_listeners.run_method( - &Listener::on_listen_finished, + report_listen_finished( + generation, "listen() failed: WSA Error Code: " + std::to_string(error) ); return; } std::string start_error; + bool superseded = false; { std::lock_guard lg(m_lock); - if (m_state.load(std::memory_order_relaxed) == State::DESTRUCTING){ - // close() won the race. - start_error = "The server was closed before it could start listening."; + superseded = m_generation != generation; + if (superseded || + m_state.load(std::memory_order_relaxed) == State::DESTRUCTING + ){ + // A newer start() or close() won the race. + start_error = "The server was replaced or closed before it could start listening."; }else{ m_listen = listen_socket; listen_socket = INVALID_SOCKET; @@ -277,6 +305,14 @@ void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t po ::closesocket(listen_socket); } + if (superseded){ + // The attempt that started this task gave up on it while it was + // still starting up (its wait timed out). A newer attempt owns + // the listeners now; reporting this result would only confuse + // that attempt, so exit without touching them. + return; + } + m_listeners.run_method(&Listener::on_listen_finished, start_error); if (!start_error.empty()){ return; @@ -290,7 +326,10 @@ void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t po { std::lock_guard lg(m_lock); State state = m_state.load(std::memory_order_relaxed); - if (state == State::DESTRUCTING || state == State::NOT_RUNNING){ + if (state == State::DESTRUCTING || + state == State::NOT_RUNNING || + m_generation != generation + ){ break; } listen = m_listen; diff --git a/Common/Cpp/Sockets/ServerSocket_WinSocket.h b/Common/Cpp/Sockets/ServerSocket_WinSocket.h index 48a2a1baf0..e12074c2af 100644 --- a/Common/Cpp/Sockets/ServerSocket_WinSocket.h +++ b/Common/Cpp/Sockets/ServerSocket_WinSocket.h @@ -8,6 +8,7 @@ #define PokemonAutomation_ServerSocket_WinSocket_H #include +#include #include #include #include @@ -61,9 +62,15 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{ // other parts of the process (Qt) may be using Winsock as well. static void ensure_wsa_started(); - void accept_thread_loop(std::string address, uint16_t port); + void accept_thread_loop(std::string address, uint16_t port, uint64_t generation); void connection_thread_loop(); + // "on_listen_finished" callback, but only for a task that still owns + // the server. A task whose generation is stale (a newer start() ran + // while it was still starting up) must stay silent, or its result + // would corrupt the attempt that replaced it. + void report_listen_finished(uint64_t generation, const std::string& error); + private: ThreadPool& m_thread_pool; @@ -72,17 +79,28 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{ bool m_starting = false; int m_closing = 0; - // Protects m_accept_thread and m_client_thread. Kept separate from - // m_lock: assigning to a task slot destroys the previous task's - // AsyncTask, and its destructor waits for that task to finish - which - // requires m_lock, so the two locks must never be held at once. + // Increments on every start(). The accept task captures its own + // generation so that a task left behind by a timed-out attempt can + // tell it was replaced when it finally gets scheduled. + uint64_t m_generation = 0; + + // Protects m_accept_threads and m_client_thread. Kept separate from + // m_lock because the task's lifetime interacts with it: joining a + // task waits for it to exit, and exiting requires m_lock. Holding + // m_lock while a task slot is destroyed would deadlock, so slot + // operations happen under m_task_lock with m_lock released. Mutex m_task_lock; SOCKET m_listen = INVALID_SOCKET; SOCKET m_client = INVALID_SOCKET; bool m_close_requested = false; - AsyncTask m_accept_thread; + // One task per start() call. start() must never destroy the previous + // task's AsyncTask here: its destructor blocks until the task has + // finished, and a task from an earlier attempt may still be running + // (the caller's wait timed out while the pool was starved). close() + // joins the whole list. + std::vector m_accept_threads; AsyncTask m_client_thread; }; diff --git a/Common/Cpp/Sockets/Sockets_Tests.cpp b/Common/Cpp/Sockets/Sockets_Tests.cpp index a3544bea42..d4e2c014e9 100644 --- a/Common/Cpp/Sockets/Sockets_Tests.cpp +++ b/Common/Cpp/Sockets/Sockets_Tests.cpp @@ -169,16 +169,26 @@ class TestSockets : public UnitTest{ protected: bool start_server(ServerSocketType& server, ServerListener& listener, uint16_t& port) const{ + // "listen_finished" is never reset between attempts, so each + // attempt records where the counter stands and waits for it to + // advance. Waiting on "> 0" instead would return immediately on a + // later attempt - with the previous attempt's stale result - and + // the loop could hand back a port the server is not actually + // listening on (the previous attempt's start() was rejected + // because the server was already running on the other port). for (int attempt = 0; attempt < 8; attempt++){ uint16_t candidate; { std::lock_guard lg(m_rng_lock); candidate = (uint16_t)(40000 + m_rng() % 20000); } + const int finished_before = listener.listen_finished.load(); server.start("127.0.0.1", candidate); if (!listener.waiter.wait_for( SOCKET_TEST_WAIT, - [&listener]{ return listener.listen_finished.load() > 0; } + [finished_before, &listener]{ + return listener.listen_finished.load() > finished_before; + } )){ continue; }