From 9129837c55191c61241a96c499e3b18e6778925f Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 20 May 2021 08:39:21 +0200 Subject: [PATCH 1/4] mxshadowsrc: Hoist two variables into static context For prefork we need the listen_socket and the filename in the threads, so hoist these variables up into the static context. --- mxshadowsrv.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mxshadowsrv.c b/mxshadowsrv.c index 9d1b423..a867fa5 100644 --- a/mxshadowsrv.c +++ b/mxshadowsrv.c @@ -74,6 +74,8 @@ static char *shadow_buf = NULL; // protected by shadow_mutex static struct stat statbuf; // protected by shadow_mutex static SSL_CTX *ssl_ctx; static sem_t free_worker; +static int listen_socket; +static char *filename; #ifdef DEBUG_MAX_CONNECTS static int debug_remaining_connects = DEBUG_MAX_CONNECTS; #endif @@ -239,7 +241,7 @@ int main(int argc, char **argv) { die_usage(argv[0]); if (optind+1 != argc) die_usage(argv[0]); - char *filename = argv[optind++]; + filename = argv[optind++]; SSL_CTX *_ssl_ctx _cleanup_(free_ssl_ctx) = SSL_CTX_new(TLS_server_method()); if (_ssl_ctx == NULL) { psslerror("SSL_CTX_new"); return 1; } @@ -248,7 +250,7 @@ int main(int argc, char **argv) { if (SSL_CTX_use_PrivateKey_file(ssl_ctx, key_file, SSL_FILETYPE_PEM) <= 0 ) { psslerror("SSL_CTX_use_PrivateKey_file"); return 1; } if (SSL_CTX_use_certificate_file(ssl_ctx, cert_file, SSL_FILETYPE_PEM) <= 0) { psslerror("SSL_CTX_use_certificate_file"); return 1; } - int listen_socket = socket(AF_INET, SOCK_STREAM, 0); + listen_socket = socket(AF_INET, SOCK_STREAM, 0); if (listen_socket == -1) { perror("socket"); return 1; } static int true = 1; From 89843f1eca65678cf7e751f7c2a636ee9d1d8b41 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 20 May 2021 09:49:51 +0200 Subject: [PATCH 2/4] mxshadowsrc: Remove DEBUG_MAX_CONNECTS Remove debug termination logic, which will be reimplemented in a different way when we prefork. --- mxshadowsrv.c | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/mxshadowsrv.c b/mxshadowsrv.c index a867fa5..5eb2ed1 100644 --- a/mxshadowsrv.c +++ b/mxshadowsrv.c @@ -7,10 +7,6 @@ #include #include #include -#include -#ifdef DEBUG_MAX_CONNECTS -#include -#endif #include "common.h" @@ -76,9 +72,6 @@ static SSL_CTX *ssl_ctx; static sem_t free_worker; static int listen_socket; static char *filename; -#ifdef DEBUG_MAX_CONNECTS -static int debug_remaining_connects = DEBUG_MAX_CONNECTS; -#endif static void validate_shadow(char **shadow_buf, char *filename, struct stat *statbuf) { int status = pthread_mutex_lock(&shadow_mutex); @@ -271,10 +264,6 @@ int main(int argc, char **argv) { if (status) { errno = status; perror("sem_init"); exit(1); } while (1) { -#ifdef DEBUG_MAX_CONNECTS - if (debug_remaining_connects-- == 0) - break; -#endif int _cleanup_(free_fd) socket = accept4(listen_socket, NULL, NULL, SOCK_NONBLOCK); if (socket == -1 ) { perror("accept"); exit(1); } status = sem_wait(&free_worker); @@ -289,16 +278,5 @@ int main(int argc, char **argv) { socket = -1; status = pthread_detach(thread); if (status != 0) { errno = status; perror("pthread_detach"); exit(1); } - - } -#ifdef DEBUG_MAX_CONNECTS - for (int i=0; i Date: Thu, 20 May 2021 08:41:33 +0200 Subject: [PATCH 3/4] mxshadowsrv: Prefork threads Prefork threads to avoid overhead per client connect. - Rename MAX_THREADS to THREADS - Hoist accept loop into client threads - Do not detach threads, use join instead --- mxshadowsrv.c | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/mxshadowsrv.c b/mxshadowsrv.c index 5eb2ed1..a90bde6 100644 --- a/mxshadowsrv.c +++ b/mxshadowsrv.c @@ -62,14 +62,13 @@ static void unmap_shadow(char *shadow_buf, struct stat *statbuf) { if (sts == -1) { perror("munmap"); exit(1); } } -#define MAX_THREADS 8 +#define THREADS 8 #define TIMEOUT 30000 // client timeout in msec static pthread_mutex_t shadow_mutex = PTHREAD_MUTEX_INITIALIZER ; static char *shadow_buf = NULL; // protected by shadow_mutex static struct stat statbuf; // protected by shadow_mutex static SSL_CTX *ssl_ctx; -static sem_t free_worker; static int listen_socket; static char *filename; @@ -188,12 +187,13 @@ static void process_client(int socket) { } static void *client_thread(void *arg) { - int socket = *((int *)arg); - process_client(socket); - close(socket); - free(arg); - int status = sem_post(&free_worker); - if (status != 0) { errno = status; perror("sem_post"); exit(1); } + + while (1) { + int _cleanup_(free_fd) socket = accept4(listen_socket, NULL, NULL, SOCK_NONBLOCK); + if (socket == -1 ) { perror("accept"); exit(1); } + validate_shadow(&shadow_buf, filename, &statbuf); + process_client(socket); + } return NULL; } @@ -260,23 +260,13 @@ int main(int argc, char **argv) { status = listen(listen_socket, 40); if (status == -1) { perror("listen"); exit(1); } - status = sem_init(&free_worker, 0, MAX_THREADS); - if (status) { errno = status; perror("sem_init"); exit(1); } - - while (1) { - int _cleanup_(free_fd) socket = accept4(listen_socket, NULL, NULL, SOCK_NONBLOCK); - if (socket == -1 ) { perror("accept"); exit(1); } - status = sem_wait(&free_worker); - if (status == -1) { perror("sem_wait"); exit(1); } - validate_shadow(&shadow_buf, filename, &statbuf); - pthread_t thread; - int *arg = malloc(sizeof *arg); - if (arg == NULL) { perror("malloc"); exit(1); } - *arg = socket; - status = pthread_create(&thread, NULL, client_thread, arg); + pthread_t thread[THREADS]; + for (int i=0; i Date: Thu, 20 May 2021 09:17:53 +0200 Subject: [PATCH 4/4] mxshadowsrv: Reintroduce DEBUG_MAX_CONNECTS --- mxshadowsrv.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mxshadowsrv.c b/mxshadowsrv.c index a90bde6..7e4614b 100644 --- a/mxshadowsrv.c +++ b/mxshadowsrv.c @@ -71,6 +71,9 @@ static struct stat statbuf; // protected by shadow_mutex static SSL_CTX *ssl_ctx; static int listen_socket; static char *filename; +#ifdef DEBUG_MAX_CONNECTS +static int debug_remaining_connects = DEBUG_MAX_CONNECTS; +#endif static void validate_shadow(char **shadow_buf, char *filename, struct stat *statbuf) { int status = pthread_mutex_lock(&shadow_mutex); @@ -189,6 +192,10 @@ static void process_client(int socket) { static void *client_thread(void *arg) { while (1) { +#ifdef DEBUG_MAX_CONNECTS + if ( __sync_fetch_and_sub(&debug_remaining_connects, 1) <= 0) + return NULL; +#endif int _cleanup_(free_fd) socket = accept4(listen_socket, NULL, NULL, SOCK_NONBLOCK); if (socket == -1 ) { perror("accept"); exit(1); } validate_shadow(&shadow_buf, filename, &statbuf);