Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Common/Cpp/Sockets/AbstractServerSocket.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ class AbstractServerSocket{

// Called when the active connection is lost (the client closed
// the connection or a socket error occurred). It is not called
// by "close()" or "close_connection()".
// by "close_and_wait()" or "close_connection()".
virtual void on_disconnect(const std::string& error_message){}
};

Expand Down Expand Up @@ -96,8 +96,8 @@ class AbstractServerSocket{
virtual void close_connection() = 0;

// Sends data to the active connection. This blocks until every byte
// has been written, the connection is lost, "close()"/
// "close_connection()" is called, or "timeout" elapses.
// has been written, the connection is lost,
// "close_and_wait()"/"close_connection()" is called, or "timeout" elapses.
// "std::chrono::milliseconds::max()" (the default) waits indefinitely.
// Returns the number of bytes actually written, which may be less
// than "bytes" if the deadline elapsed, the connection was lost, or
Expand Down
8 changes: 4 additions & 4 deletions Common/Cpp/Sockets/ServerSocket_POSIX.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ void ServerSocket_POSIX::start(const std::string& address, uint16_t port){
// caller's wait timed out while the pool was starved). It
// notices its stale generation when it gets scheduled and
// exits without listening, so it only has to outlive this
// call; close() joins the whole list.
// call; close_and_wait() joins the whole list.
std::lock_guard<Mutex> lg(m_task_lock);
m_accept_threads.push_back(std::move(task));
}
Expand Down Expand Up @@ -101,7 +101,7 @@ void ServerSocket_POSIX::close_connection(){
}
}

void ServerSocket_POSIX::close() noexcept{
void ServerSocket_POSIX::close_and_wait() noexcept{
{
std::unique_lock<Mutex> lg(m_lock);
// Wait for an in-progress start() to publish the listener task
Expand All @@ -121,7 +121,7 @@ void ServerSocket_POSIX::close() noexcept{
}
if (!drain){
// Either nothing was running (m_closing == 0), or another
// close() is already draining the threads (m_closing > 1).
// close_and_wait() is already draining the threads (m_closing > 1).
// In the second case, wait for it to finish.
std::unique_lock<Mutex> lg(m_lock);
m_cv.wait(lg, [&]{
Expand Down Expand Up @@ -280,7 +280,7 @@ void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port,
if (superseded ||
m_state.load(std::memory_order_relaxed) == State::DESTRUCTING
){
// A newer start() or close() won the race.
// A newer start() or close_and_wait() won the race.
start_error = "The server was replaced or closed before it could start listening.";
}else{
m_listen = listen_socket;
Expand Down
8 changes: 4 additions & 4 deletions Common/Cpp/Sockets/ServerSocket_POSIX.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class ServerSocket_POSIX final : public AbstractServerSocket{
ServerSocket_POSIX(ThreadPool& thread_pool);

virtual ~ServerSocket_POSIX(){
close();
close_and_wait();
}

virtual void start(const std::string& address, uint16_t port) override;
Expand All @@ -52,7 +52,7 @@ class ServerSocket_POSIX final : public AbstractServerSocket{
// active connection, and returns the server to State::NOT_RUNNING so
// the object is fully torn down. It blocks until the worker threads
// have exited, so it is private and only called by the destructor.
void close() noexcept;
void close_and_wait() noexcept;

void accept_thread_loop(std::string address, uint16_t port, uint64_t generation);
void connection_thread_loop();
Expand Down Expand Up @@ -90,8 +90,8 @@ class ServerSocket_POSIX final : public AbstractServerSocket{
// One task per start() call. start() must never destroy the previous
// task's AsyncTask here: its destructor blocks until the task has
// finished, and a task from an earlier attempt may still be running
// (the caller's wait timed out while the pool was starved). close()
// joins the whole list.
// (the caller's wait timed out while the pool was starved).
// close_and_wait() joins the whole list.
std::vector<AsyncTask> m_accept_threads;
AsyncTask m_client_thread;
};
Expand Down
40 changes: 21 additions & 19 deletions Common/Cpp/Sockets/ServerSocket_Qt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ void ServerSocket_Qt::start(const std::string& address, uint16_t port){
&& !m_starting;
// Only claim the "starting" flag when we actually start: a
// rejected call (server already running or already started) must
// not leave "m_starting" true - "close()" waits on
// not leave "m_starting" true - "close_and_wait()" waits on
// "!m_starting", so a stale flag would deadlock the destructor.
// (This happens when a test's start-retry loop calls start() on
// a server whose first listen failed: "m_server" is set even on
Expand Down Expand Up @@ -66,7 +66,7 @@ void ServerSocket_Qt::start(const std::string& address, uint16_t port){
(m_state.load(std::memory_order_relaxed) == State::DESTRUCTING);
m_server = server;
if (destructing){
// "close()" is already in progress: it will destroy the
// "close_and_wait()" is already in progress: it will destroy the
// server we just created. Leave the state for it.
}else{
listening = m_listen_error.empty();
Expand All @@ -83,7 +83,7 @@ void ServerSocket_Qt::start(const std::string& address, uint16_t port){
// exactly as it was before this start() (no "m_server", state
// NOT_RUNNING), so the caller's start-retry loop can immediately
// try another port. The other backends behave the same: a failed
// listen leaves no listener behind. If "close()" raced us, it has
// listen leaves no listener behind. If "close_and_wait()" raced us, it has
// already claimed "m_server", and the pool's lookup makes the
// second destruction a no-op.
GlobalThreadPools::qt_event_threadpool().remove_object(server);
Expand Down Expand Up @@ -130,7 +130,7 @@ void ServerSocket_Qt::close_connection(){
);
}
}
void ServerSocket_Qt::close() noexcept{
void ServerSocket_Qt::close_and_wait() noexcept{
QTcpServer* server;
{
std::lock_guard<Mutex> lg(m_lock);
Expand Down Expand Up @@ -195,12 +195,13 @@ size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::millis
? std::chrono::steady_clock::time_point::max()
: std::chrono::steady_clock::now() + timeout;

// Register the send atomically with respect to "close()"/
// "close_connection()": "close()" sets the state to DESTRUCTING and
// detaches the socket under this same lock before it cancels the
// in-flight sends. The packet copy above can take a while for large
// payloads, so without this re-check a send could register *after*
// "close()" already cancelled the (empty) list and queued the
// Register the send atomically with respect to
// "close_and_wait()"/"close_connection()": "close_and_wait()" sets the
// state to DESTRUCTING and detaches the socket under this same lock
// before it cancels the in-flight sends. The packet copy above can
// take a while for large payloads, so without this re-check a send
// could register *after*
// "close_and_wait()" already cancelled the (empty) list and queued the
// server's destruction: its "internal_send" call would then be
// posted after the destruction event, Qt would silently discard it,
// and the wait below (unbounded for the infinite default timeout)
Expand All @@ -224,17 +225,17 @@ size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::millis
emit internal_send(send_data);

// Wait for the server's event thread to finish writing the packet,
// or for "close()"/"close_connection()" to cancel the send. A
// or for "close_and_wait()"/"close_connection()" to cancel the send. A
// finite "timeout" waits for the deadline plus 1 s of queueing
// margin: the write loop stops at the deadline, so anything beyond
// that is just the queued call waiting to start.
//
// The infinite default waits without a time bound. The registration
// above is atomic with the "close()"/"close_connection()" detach, so
// above is atomic with the "close_and_wait()"/"close_connection()" detach, so
// "done" is set by either the event-thread write loop (which stops
// at the deadline, on a socket error, or when Qt's userspace buffer
// accepts the packet; it runs before the queued server destruction,
// since the event loop is FIFO) or by "close()"/"close_connection()"
// since the event loop is FIFO) or by "close_and_wait()"/"close_connection()"
// cancelling the registered send - even when Qt discards the queued
// write loop with the server - so the wait always ends. The shared
// ownership keeps the packet alive until the queued call runs (or
Expand Down Expand Up @@ -289,8 +290,8 @@ void ServerSocket_Qt::on_new_connection(){
if (sock == nullptr){
break;
}
// The connection socket lives with the server, so "close()"
// destroys it on the event thread.
// The connection socket lives with the server, so
// "close_and_wait()" destroys it on the event thread.
sock->setParent(server);

bool accepted;
Expand Down Expand Up @@ -393,7 +394,7 @@ void ServerSocket_Qt::on_close_connection(QSharedPointer<CloseCompletion> comple
m_close_requested = true;
if (m_state.load(std::memory_order_relaxed) != State::DESTRUCTING){
// The server keeps running, so it goes back to
// LISTENING. The destructor's close() may have set
// LISTENING. The destructor's close_and_wait() may have set
// DESTRUCTING: leave the state for it.
m_state.store(State::LISTENING, std::memory_order_release);
}
Expand Down Expand Up @@ -424,7 +425,7 @@ void ServerSocket_Qt::on_send(QSharedPointer<SendData> send_data){
{
std::lock_guard<Mutex> lg(send_data->lock);
if (send_data->cancelled){
// "close()"/"close_connection()" already cancelled this
// "close_and_wait()"/"close_connection()" already cancelled this
// send and woken its caller; finish without writing.
return;
}
Expand All @@ -435,7 +436,8 @@ void ServerSocket_Qt::on_send(QSharedPointer<SendData> send_data){
const char* ptr = send_data->packet.data();
size_t remaining = send_data->packet.size();
// The deadline keeps a client that never reads from pinning this
// event thread forever (which would also block "close()").
// event thread forever (which would also block
// "close_and_wait()").
while (
remaining > 0
&& sock->state() == QAbstractSocket::ConnectedState
Expand Down Expand Up @@ -470,7 +472,7 @@ void ServerSocket_Qt::on_send(QSharedPointer<SendData> send_data){

{
std::lock_guard<Mutex> lg(m_lock);
// "close()"/"close_connection()" may already have claimed this
// "close_and_wait()"/"close_connection()" may already have claimed this
// send; only remove it if it still owns its slot.
m_inflight_sends.erase(
std::remove(
Expand Down
25 changes: 14 additions & 11 deletions Common/Cpp/Sockets/ServerSocket_Qt.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,26 +31,27 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{
Q_OBJECT

// Carries shared ownership (the signals pass it by value) so that a call still
// queued when the caller times out, or when "close()" destroys the server and
// queued when the caller times out, or when "close_and_wait()" destroys the server and
// Qt discards the pending call, keeps the payload alive exactly as long as the
// event itself: the handler's copy outlives a timed-out caller, and a
// discarded event leaves the caller's copy as the last reference.
struct SendData{
std::string packet;
// The event-thread write loop stops at this deadline, so a send
// that can never complete (a client that never reads) cannot pin
// the server's event thread forever and block "close()". "max()"
// means no deadline (the infinite default).
// the server's event thread forever and block
// "close_and_wait()". "max()" means no deadline (the infinite
// default).
std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::time_point::max();
// The running byte count: the write loop updates it as it goes,
// so "close()"/"close_connection()" cancelling the send mid-write
// can report the bytes written so far (best effort).
// so "close_and_wait()"/"close_connection()" cancelling the send
// mid-write can report the bytes written so far (best effort).
size_t bytes_sent = 0;
// Set by "close()"/"close_connection()" when they cancel the
// Set by "close_and_wait()"/"close_connection()" when they cancel the
// send; the queued write loop then finishes without writing.
bool cancelled = false;
// Set when the event-thread write loop finishes or when
// "close()"/"close_connection()" cancel the send.
// "close_and_wait()"/"close_connection()" cancel the send.
bool done = false;

Mutex lock;
Expand All @@ -69,7 +70,7 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{
ServerSocket_Qt(ThreadPool&){}

virtual ~ServerSocket_Qt(){
close();
close_and_wait();
}

// Binds to "address" and listens on "port" on the Qt event thread. This
Expand Down Expand Up @@ -101,7 +102,8 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{
// The active connection ended, or was aborted by "close_connection()".
// Only the first of the (errorOccurred, disconnected) signals for a
// given connection fires a callback, and "on_disconnect" is not fired
// for a connection closed by "close_connection()" or "close()".
// for a connection closed by "close_connection()" or
// "close_and_wait()".
// Runs on the QTcpServer's event thread.
void connection_lost(QTcpSocket* sock, const std::string& error_message);

Expand All @@ -126,7 +128,7 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{
// until the QTcpServer has been destroyed, so any in-flight listener
// callback is finished by the time it returns; it is private and
// only called by the destructor.
void close() noexcept;
void close_and_wait() noexcept;

std::unique_ptr<QTcpServer> make_server(const std::string& address, uint16_t port);

Expand All @@ -141,7 +143,8 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{
bool m_starting = false;

// The "send()" calls that are queued on the event thread but not
// finished yet, so that "close()"/"close_connection()" can cancel
// finished yet, so that "close_and_wait()"/"close_connection()" can
// cancel
// them (and wake their callers) before the socket is destroyed. The
// write loops remove their entry when they finish. Protected by
// m_lock.
Expand Down
8 changes: 4 additions & 4 deletions Common/Cpp/Sockets/ServerSocket_WinSocket.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ void ServerSocket_WinSocket::start(const std::string& address, uint16_t port){
// caller's wait timed out while the pool was starved). It
// notices its stale generation when it gets scheduled and
// exits without listening, so it only has to outlive this
// call; close() joins the whole list.
// call; close_and_wait() joins the whole list.
std::lock_guard<Mutex> lg(m_task_lock);
m_accept_threads.push_back(std::move(task));
}
Expand Down Expand Up @@ -112,7 +112,7 @@ void ServerSocket_WinSocket::close_connection(){
}
}

void ServerSocket_WinSocket::close() noexcept{
void ServerSocket_WinSocket::close_and_wait() noexcept{
{
std::unique_lock<Mutex> lg(m_lock);
// Wait for an in-progress start() to publish the listener task
Expand All @@ -132,7 +132,7 @@ void ServerSocket_WinSocket::close() noexcept{
}
if (!drain){
// Either nothing was running (m_closing == 0), or another
// close() is already draining the threads (m_closing > 1).
// close_and_wait() is already draining the threads (m_closing > 1).
// In the second case, wait for it to finish.
std::unique_lock<Mutex> lg(m_lock);
m_cv.wait(lg, [&]{
Expand Down Expand Up @@ -293,7 +293,7 @@ void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t po
if (superseded ||
m_state.load(std::memory_order_relaxed) == State::DESTRUCTING
){
// A newer start() or close() won the race.
// A newer start() or close_and_wait() won the race.
start_error = "The server was replaced or closed before it could start listening.";
}else{
m_listen = listen_socket;
Expand Down
8 changes: 4 additions & 4 deletions Common/Cpp/Sockets/ServerSocket_WinSocket.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{
: m_thread_pool(thread_pool)
{}
virtual ~ServerSocket_WinSocket(){
close();
close_and_wait();
}

virtual void start(const std::string& address, uint16_t port) override;
Expand Down Expand Up @@ -61,7 +61,7 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{
// active connection, and returns the server to State::NOT_RUNNING so
// the object is fully torn down. It blocks until the worker threads
// have exited, so it is private and only called by the destructor.
void close() noexcept;
void close_and_wait() noexcept;

void accept_thread_loop(std::string address, uint16_t port, uint64_t generation);
void connection_thread_loop();
Expand Down Expand Up @@ -99,8 +99,8 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{
// One task per start() call. start() must never destroy the previous
// task's AsyncTask here: its destructor blocks until the task has
// finished, and a task from an earlier attempt may still be running
// (the caller's wait timed out while the pool was starved). close()
// joins the whole list.
// (the caller's wait timed out while the pool was starved).
// close_and_wait() joins the whole list.
std::vector<AsyncTask> m_accept_threads;
AsyncTask m_client_thread;
};
Expand Down
Loading