Fix: sbd-integration: sync pacemakerd with sbd

daemons/pacemakerd/pacemakerd.c

@@ -40,8 +40,25 @@ static bool global_keep_tracking = false;
 #define PCMK_PROCESS_CHECK_INTERVAL 5
 
 static crm_trigger_t *shutdown_trigger = NULL;
+static crm_trigger_t *startup_trigger = NULL;
 static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid";
 
+/* state we report when asked via pacemakerd-api status-ping */
+static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
+static gboolean running_with_sbd = FALSE; /* local copy */
+/* When contacted via pacemakerd-api by a client having sbd in
+ * the name we assume it is sbd-daemon which wants to know
+ * if pacemakerd shutdown gracefully.
+ * Thus when everything is shutdown properly pacemakerd
+ * waits till it has reported the graceful completion of
+ * shutdown to sbd and just when sbd-client closes the
+ * connection we can assume that the report has arrived
+ * properly so that pacemakerd can finally exit.
+ * Following two variables are used to track that handshake.
+ */
+static unsigned int shutdown_complete_state_reported_to = 0;
+static gboolean shutdown_complete_state_reported_client_closed = FALSE;
+
 typedef struct pcmk_child_s {
     pid_t pid;
     long flag;
@@ -374,21 +391,20 @@ escalate_shutdown(gpointer data)
 static gboolean
 pcmk_shutdown_worker(gpointer user_data)
 {
-    static int phase = 0;
+    static int phase = SIZEOF(pcmk_children);
     static time_t next_log = 0;
-    static int max = SIZEOF(pcmk_children);
 
     int lpc = 0;
 
-    if (phase == 0) {
+    if (phase == SIZEOF(pcmk_children)) {
         crm_notice("Shutting down Pacemaker");
-        phase = max;
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
     }
 
     for (; phase > 0; phase--) {
         /* Don't stop anything with start_seq < 1 */
 
-        for (lpc = max - 1; lpc >= 0; lpc--) {
+        for (lpc = SIZEOF(pcmk_children) - 1; lpc >= 0; lpc--) {
             pcmk_child_t *child = &(pcmk_children[lpc]);
 
             if (phase != child->start_seq) {
@@ -436,6 +452,13 @@ pcmk_shutdown_worker(gpointer user_data)
     }
 
     crm_notice("Shutdown complete");
+    pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
+    if (!fatal_error && running_with_sbd &&
+        pcmk__get_sbd_sync_resource_startup() &&
+        !shutdown_complete_state_reported_client_closed) {
+        crm_notice("Waiting for SBD to pick up shutdown-complete-state.");
+        return TRUE;
+    }
 
     {
         const char *delay = pcmk__env_option("shutdown_delay");
@@ -489,6 +512,55 @@ pcmk_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     return 0;
 }
 
+static void
+pcmk_handle_ping_request(pcmk__client_t *c, xmlNode *msg, uint32_t id)
+{
+    const char *value = NULL;
+    xmlNode *ping = NULL;
+    xmlNode *reply = NULL;
+    time_t pinged = time(NULL);
+    const char *from = crm_element_value(msg, F_CRM_SYS_FROM);
+
+    /* Pinged for status */
+    crm_trace("Pinged from %s.%s",
+              crm_str(crm_element_value(msg, F_CRM_ORIGIN)),
+              from?from:"unknown");
+    ping = create_xml_node(NULL, XML_CRM_TAG_PING);
+    value = crm_element_value(msg, F_CRM_SYS_TO);
+    crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
+    crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state);
+    crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged);
+    crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
+    reply = create_reply(msg, ping);
+    free_xml(ping);
+    if (reply) {
+        if (pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event) !=
+                pcmk_rc_ok) {
+            crm_err("Failed sending ping-reply");
+        }
+        free_xml(reply);
+    } else {
+        crm_err("Failed building ping-reply");
+    }
+    /* just proceed state on sbd pinging us */
+    if (from && strstr(from, "sbd")) {
+        if (crm_str_eq(pacemakerd_state,
+                       XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE,
+                       TRUE)) {
+            if (pcmk__get_sbd_sync_resource_startup()) {
+                crm_notice("Shutdown-complete-state passed to SBD.");
+            }
+            shutdown_complete_state_reported_to = c->pid;
+        } else if (crm_str_eq(pacemakerd_state,
+                              XML_PING_ATTR_PACEMAKERDSTATE_WAITPING,
+                              TRUE)) {
+            crm_notice("Received startup-trigger from SBD.");
+            pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
+            mainloop_set_trigger(startup_trigger);
+        }
+    }
+}
+
 /* Exit code means? */
 static int32_t
 pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
@@ -514,6 +586,9 @@ pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
         crm_trace("Ignoring IPC request to purge node "
                   "because peer cache is not used");
 
+    } else if (crm_str_eq(task, CRM_OP_PING, TRUE)) {
+        pcmk_handle_ping_request(c, msg, id);
+
     } else {
         crm_debug("Unrecognized IPC command '%s' sent to pacemakerd",
                   crm_str(task));
@@ -533,6 +608,12 @@ pcmk_ipc_closed(qb_ipcs_connection_t * c)
         return 0;
     }
     crm_trace("Connection %p", c);
+    if (shutdown_complete_state_reported_to == client->pid) {
+        shutdown_complete_state_reported_client_closed = TRUE;
+        if (shutdown_trigger) {
+            mainloop_set_trigger(shutdown_trigger);
+        }
+    }
     pcmk__free_client(client);
     return 0;
 }
@@ -924,8 +1005,8 @@ find_and_track_existing_processes(void)
     return pcmk_rc_ok;
 }
 
-static void
-init_children_processes(void)
+static gboolean
+init_children_processes(void *user_data)
 {
     int start_seq = 1, lpc = 0;
     static int max = SIZEOF(pcmk_children);
@@ -951,6 +1032,8 @@ init_children_processes(void)
      * This may be useful for the daemons to know
      */
     setenv("PCMK_respawned", "true", 1);
+    pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
+    return TRUE;
 }
 
 static void
@@ -1154,6 +1237,7 @@ main(int argc, char **argv)
 
     if(pcmk_locate_sbd() > 0) {
         setenv("PCMK_watchdog", "true", 1);
+        running_with_sbd = TRUE;
     } else {
         setenv("PCMK_watchdog", "false", 1);
     }
@@ -1170,7 +1254,19 @@ main(int argc, char **argv)
     mainloop_add_signal(SIGTERM, pcmk_shutdown);
     mainloop_add_signal(SIGINT, pcmk_shutdown);
 
-    init_children_processes();
+    if ((running_with_sbd) && pcmk__get_sbd_sync_resource_startup()) {
+        crm_notice("Waiting for startup-trigger from SBD.");
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING;
+        startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL);
+    } else {
+        if (running_with_sbd) {
+            crm_warn("Enabling SBD_SYNC_RESOURCE_STARTUP would (if supported "
+                     "by your SBD version) improve reliability of "
+                     "interworking between SBD & pacemaker.");
+        }
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
+        init_children_processes(NULL);
+    }
 
     crm_notice("Pacemaker daemon successfully started and accepting connections");
     g_main_loop_run(mainloop);

include/crm/common/Makefile.am

@@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in
 headerdir=$(pkgincludedir)/crm/common
 
 header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \
-		 nvpair.h acl.h ipc_controld.h
+		 nvpair.h acl.h ipc_controld.h ipc_pacemakerd.h
 noinst_HEADERS = internal.h alerts_internal.h \
 		 iso8601_internal.h remote_internal.h xml_internal.h \
 		 ipc_internal.h output.h cmdline_internal.h curses_internal.h \

include/crm/common/ipc_pacemakerd.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PCMK__IPC_PACEMAKERD__H
+#  define PCMK__IPC_PACEMAKERD__H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \file
+ * \brief IPC commands for Pacemakerd
+ *
+ * \ingroup core
+ */
+
+#include <sys/types.h>       // time_t
+#include <crm/common/ipc.h>  // pcmk_ipc_api_t
+
+enum pcmk_pacemakerd_state {
+    pcmk_pacemakerd_state_invalid = -1,
+    pcmk_pacemakerd_state_init = 0,
+    pcmk_pacemakerd_state_starting_daemons,
+    pcmk_pacemakerd_state_wait_for_ping,
+    pcmk_pacemakerd_state_running,
+    pcmk_pacemakerd_state_shutting_down,
+    pcmk_pacemakerd_state_shutdown_complete,
+    pcmk_pacemakerd_state_max = pcmk_pacemakerd_state_shutdown_complete,
+};
+
+//! Possible types of pacemakerd replies
+enum pcmk_pacemakerd_api_reply {
+    pcmk_pacemakerd_reply_unknown,
+    pcmk_pacemakerd_reply_ping,
+};
+
+/*!
+ * Pacemakerd reply passed to event callback
+ */
+typedef struct {
+    enum pcmk_pacemakerd_api_reply reply_type;
+
+    union {
+        // pcmk_pacemakerd_reply_ping
+        struct {
+            const char *sys_from;
+            enum pcmk_pacemakerd_state state;
+            time_t last_good;
+            int status;
+        } ping;
+    } data;
+} pcmk_pacemakerd_api_reply_t;
+
+int pcmk_pacemakerd_api_ping(pcmk_ipc_api_t *api, const char *ipc_name);
+enum pcmk_pacemakerd_state
+    pcmk_pacemakerd_api_daemon_state_text2enum(const char *state);
+const char
+    *pcmk_pacemakerd_api_daemon_state_enum2text(enum pcmk_pacemakerd_state state);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // PCMK__IPC_PACEMAKERD__H

include/crm/common/options_internal.h

@@ -111,6 +111,7 @@ bool pcmk__valid_utilization(const char *value);
 
 // from watchdog.c
 long pcmk__get_sbd_timeout(void);
+bool pcmk__get_sbd_sync_resource_startup(void);
 long pcmk__auto_watchdog_timeout(void);
 bool pcmk__valid_sbd_timeout(const char *value);
 

include/crm/crm.h

@@ -51,7 +51,7 @@ extern "C" {
  * >=3.0.13: Fail counts include operation name and interval
  * >=3.2.0:  DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED
  */
-#  define CRM_FEATURE_SET		"3.4.0"
+#  define CRM_FEATURE_SET		"3.4.1"
 
 #  define EOS		'\0'
 #  define DIMOF(a)	((int) (sizeof(a)/sizeof(a[0])) )

include/crm/msg_xml.h

@@ -123,6 +123,13 @@ extern "C" {
 #  define XML_PING_ATTR_STATUS		"result"
 #  define XML_PING_ATTR_SYSFROM		"crm_subsystem"
 #  define XML_PING_ATTR_CRMDSTATE   "crmd_state"
+#  define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete"
 
 #  define XML_TAG_FRAGMENT		"cib_fragment"
 

lib/common/Makefile.am

@@ -50,6 +50,7 @@ libcrmcommon_la_SOURCES	+= io.c
 libcrmcommon_la_SOURCES	+= ipc_client.c
 libcrmcommon_la_SOURCES	+= ipc_common.c
 libcrmcommon_la_SOURCES	+= ipc_controld.c
+libcrmcommon_la_SOURCES	+= ipc_pacemakerd.c
 libcrmcommon_la_SOURCES	+= ipc_server.c
 libcrmcommon_la_SOURCES	+= iso8601.c
 libcrmcommon_la_SOURCES	+= logging.c

lib/common/crmcommon_private.h

@@ -175,6 +175,7 @@ typedef struct pcmk__ipc_methods_s {
 struct pcmk_ipc_api_s {
     enum pcmk_ipc_server server;          // Daemon this IPC API instance is for
     enum pcmk_ipc_dispatch dispatch_type; // How replies should be dispatched
+    size_t ipc_size_max;                  // maximum IPC buffer size
     crm_ipc_t *ipc;                       // IPC connection
     mainloop_io_t *mainloop_io;     // If using mainloop, I/O source for IPC
     bool free_on_disconnect;        // Whether disconnect should free object
@@ -209,4 +210,7 @@ bool pcmk__valid_ipc_header(const pcmk__ipc_header_t *header);
 G_GNUC_INTERNAL
 pcmk__ipc_methods_t *pcmk__controld_api_methods(void);
 
+G_GNUC_INTERNAL
+pcmk__ipc_methods_t *pcmk__pacemakerd_api_methods(void);
+
 #endif  // CRMCOMMON_PRIVATE__H