mxqd: use mx_mysql_do_statement_retry_on_fail() for some queries

Makefile

@@ -84,6 +84,7 @@ LDLIBS_MYSQL := $(shell $(MYSQL_CONFIG) --libs)
 
 CFLAGS_MYSQL += ${CFLAGS_MXQ_MYSQL_DEFAULT_FILE}
 CFLAGS_MYSQL += ${CFLAGS_MXQ_MYSQL_DEFAULT_GROUP}
+CFLAGS_MYSQL += -DMX_MYSQL_FAIL_WAIT_DEFAULT=5
 
 CFLAGS += -g
 CFLAGS += -Wall

mx_mysql.c

@@ -208,6 +208,16 @@ static inline int mx__mysql_ping(struct mx_mysql *mysql)
             mx__mysql_log_emerg(mysql);
             return -(errno=EPROTO);
 
+        case CR_OUT_OF_MEMORY:
+            return -(errno=ENOMEM);
+
+        case CR_CONN_HOST_ERROR:
+        case CR_CONNECTION_ERROR:
+        case CR_IPSOCK_ERROR:
+        case CR_SOCKET_CREATE_ERROR:
+        case CR_UNKNOWN_HOST:
+        case CR_VERSION_ERROR:
+        case CR_SERVER_LOST:
         case CR_SERVER_GONE_ERROR:
             return -(errno=EAGAIN);
 
@@ -785,8 +795,8 @@ int mx_mysql_init(struct mx_mysql *mysql)
         if (res != -ENOMEM)
             return res;
 
-        mx_log_debug("mx__mysql_init() failed: %m - retrying (forever) in %d second(s).", MX_CALLOC_FAIL_WAIT_DEFAULT);
-        mx_sleep(MX_CALLOC_FAIL_WAIT_DEFAULT);
+        mx_log_debug("mx__mysql_init() failed: %m - retrying (forever) in %d second(s).", MX_MYSQL_FAIL_WAIT_DEFAULT);
+        mx_sleep(MX_MYSQL_FAIL_WAIT_DEFAULT);
 
     } while (1);
 
@@ -971,6 +981,33 @@ int mx_mysql_ping(struct mx_mysql *mysql)
     return mx__mysql_ping(mysql);
 }
 
+int mx_mysql_ping_forever(struct mx_mysql *mysql)
+{
+    int res;
+    int fail = 0;
+
+    mx_assert_return_minus_errno(mysql, EINVAL);
+
+    while (1) {
+        res = mx_mysql_ping(mysql);
+        if (res == 0)
+            break;
+
+        fail++;
+
+        mx__mysql_log_warning(mysql);
+        mx_mysql_assert_usage_ok(res);
+        mx_log_warning("mx_mysql_ping() failed: %m - retrying again (forever) in %d second(s).", MX_MYSQL_FAIL_WAIT_DEFAULT);
+        mx_sleep(MX_MYSQL_FAIL_WAIT_DEFAULT);
+    }
+
+    if (fail)
+        mx_log_info("mx_mysql_ping_forever() recovered from previous errors (%d tries). Yippieh! Back to work!", fail);
+
+    return res;
+}
+
+
 int mx_mysql_queryf(struct mx_mysql *mysql, const char *fmt, ...)
 {
     mx_assert_return_minus_errno(mysql, EINVAL);
@@ -1261,9 +1298,12 @@ int mx_mysql_do_statement(struct mx_mysql *mysql, char *query, struct mx_mysql_b
 
     assert(mysql);
 
+    mx_mysql_ping_forever(mysql);
+
     stmt = mx_mysql_statement_prepare_with_bindings(mysql, query, param, result);
     if (!stmt) {
-        mx_log_err("mx_mysql_statement_prepare(): %m");
+        mx_log_err("mx_mysql_statement_prepare_with_bindings(): %m");
+        mx_log_err("mx_mysql_statement_prepare_with_bindings(): query was: %s", query);
         return -errno;
     }
 
@@ -1294,6 +1334,32 @@ int mx_mysql_do_statement(struct mx_mysql *mysql, char *query, struct mx_mysql_b
     return num_rows;
 }
 
+int mx_mysql_do_statement_retry_on_fail(struct mx_mysql *mysql, char *query, struct mx_mysql_bind *param, struct mx_mysql_bind *result, void *from, void **to, size_t size)
+{
+    int res;
+
+    mx_log_debug("entered");
+
+    while (1) {
+        res = mx_mysql_do_statement(mysql, query, param, result, from, to, size);
+
+        if (res >= 0)
+            break;
+
+        mx_mysql_assert_usage_ok(res);
+
+        mx_log_warning("mx_mysql_do_statement() failed: %m");
+
+        if (res != -EAGAIN)
+            break;
+
+        mx_mysql_ping_forever(mysql);
+    }
+
+    return res;
+}
+
+
 struct mx_mysql_stmt *mx_mysql_statement_prepare_with_bindings(struct mx_mysql *mysql, char *statement, struct mx_mysql_bind *param, struct mx_mysql_bind *result)
 {
     int res;

mx_mysql.h

@@ -104,9 +104,11 @@ int mx_mysql_free(struct mx_mysql **mysql);
 
 int mx_mysql_option_set_default_file(struct mx_mysql *mysql, char *fname);
 int mx_mysql_option_set_default_group(struct mx_mysql *mysql, char *group);
+int mx_mysql_option_set_reconnect(struct mx_mysql *mysql, int reconnect);
 
 char *mx_mysql_option_get_default_file(struct mx_mysql *mysql);
 char *mx_mysql_option_get_default_group(struct mx_mysql *mysql);
+int   mx_mysql_option_get_reconnect(struct mx_mysql *mysql);
 
 int mx_mysql_connect(struct mx_mysql **mysql);
 int mx_mysql_connect_forever_sec(struct mx_mysql **mysql, unsigned int seconds);
@@ -118,9 +120,18 @@ int mx_mysql_end(void);
 
 int mx_mysql_finish(struct mx_mysql **mysql);
 
+int mx_mysql_ping(struct mx_mysql *mysql);
+int mx_mysql_ping_forever(struct mx_mysql *mysql);
+
+
 #define mx_mysql_do_statement_noresult(m, q, p) \
         mx_mysql_do_statement(m, q, p, NULL, NULL, NULL, 0)
+
+#define mx_mysql_do_statement_noresult_retry_on_fail(m, q, p) \
+        mx_mysql_do_statement_retry_on_fail(m, q, p, NULL, NULL, NULL, 0)
+
 int mx_mysql_do_statement(struct mx_mysql *mysql, char *query, struct mx_mysql_bind *param, struct mx_mysql_bind *result, void *from, void **to, size_t size);
+int mx_mysql_do_statement_retry_on_fail(struct mx_mysql *mysql, char *query, struct mx_mysql_bind *param, struct mx_mysql_bind *result, void *from, void **to, size_t size);
 
 int mx_mysql_statement_init(struct mx_mysql *mysql, struct mx_mysql_stmt **stmt);
 struct mx_mysql_stmt *mx_mysql_statement_prepare(struct mx_mysql *mysql, char *statement);

mxq_group.c

@@ -303,9 +303,9 @@ int mxq_load_active_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups
     res = bind_result_group_fields(&result, &g);
     assert(res == 0);
 
-    res = mx_mysql_do_statement(mysql, query, NULL, &result, &g, (void **)&groups, sizeof(*groups));
+    res = mx_mysql_do_statement_retry_on_fail(mysql, query, NULL, &result, &g, (void **)&groups, sizeof(*groups));
     if (res < 0) {
-        mx_log_err("mx_mysql_do_statement(): %m");
+        mx_log_err("mx_mysql_do_statement_retry_on_fail(): %m");
         return res;
     }
 

mxqd.c

@@ -324,6 +324,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
 
     mx_mysql_option_set_default_file(server->mysql,  arg_mysql_default_file);
     mx_mysql_option_set_default_group(server->mysql, arg_mysql_default_group);
+    mx_mysql_option_set_reconnect(server->mysql, 1);
 
     server->hostname = mxq_hostname();
     server->server_id = arg_server_id;