From adeeece59abbc5ed1646c9b52c8e1032bd54460f Mon Sep 17 00:00:00 2001 From: Rick Falkvinge Date: Mon, 16 Jan 2017 22:09:09 +0100 Subject: [PATCH] Adding a lot of heartbeating and OOBE work --- Backend/Backend.csproj | 3 +- Backend/BackendLoop.cs | 15 +- Backend/etcinitd/etcinitd-swarmops-backend | 164 ----------------- Backend/etcinitd/swarmops-backend | 170 ------------------ Backend/libsystemd/swarmops-backend.service | 3 +- Common/ExtensionMethods/DateTime.cs | 4 +- FrontSocket/Frontend-Socket.csproj | 2 +- FrontSocket/FrontendLoop.cs | 13 ++ .../libsystemd/swarmops-frontend.service | 3 +- Logic/Support/SystemSettings.cs | 59 ++++++ Site/Pages/v5/Public/Initialize.aspx | 18 +- Site/Pages/v5/Public/Initialize.aspx.cs | 31 +++- 12 files changed, 138 insertions(+), 347 deletions(-) delete mode 100644 Backend/etcinitd/etcinitd-swarmops-backend delete mode 100644 Backend/etcinitd/swarmops-backend diff --git a/Backend/Backend.csproj b/Backend/Backend.csproj index 819839dc..e1005fc5 100644 --- a/Backend/Backend.csproj +++ b/Backend/Backend.csproj @@ -121,8 +121,7 @@ - - + diff --git a/Backend/BackendLoop.cs b/Backend/BackendLoop.cs index a955156b..061b6c38 100644 --- a/Backend/BackendLoop.cs +++ b/Backend/BackendLoop.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.Threading; using Mono.Unix; using Mono.Unix.Native; +using Swarmops.Common.ExtensionMethods; using Swarmops.Database; using Swarmops.Logic.Communications; using Swarmops.Logic.Communications.Payload; @@ -25,7 +26,18 @@ namespace Swarmops.Backend private static void Main (string[] args) { - // Checking for schemata upgrade first of all + // Are we running yet? + + if (!SystemSettings.DatabaseInitialized) + { + // will restart the service every 30s until db initialized on OOBE + // also, the read of DatabaseInitialized can and will fail if + // we're not initalized enough to even have a database + + throw new InvalidOperationException(); + } + + // Checking for schemata upgrade first of all, after seeing that db exists DatabaseMaintenance.UpgradeSchemata(); @@ -315,6 +327,7 @@ namespace Swarmops.Backend { BotLog.Write (0, "MainCycle", "Ten-second entry"); + SystemSettings.HeartbeatBackend = (ulong) DateTime.UtcNow.ToUnix(); CommsTransmitter.Run(); try diff --git a/Backend/etcinitd/etcinitd-swarmops-backend b/Backend/etcinitd/etcinitd-swarmops-backend deleted file mode 100644 index 75abdbf0..00000000 --- a/Backend/etcinitd/etcinitd-swarmops-backend +++ /dev/null @@ -1,164 +0,0 @@ -#! /bin/bash -### BEGIN INIT INFO -# Provides: swarmops-backend -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Swarmops backend -# Description: This file starts and stops the Swarmsops backend. It is installed as part of -# the swarmops-backend package and placed in /etc/init.d for compatibility with -# Debian and Ubuntu servers. -### END INIT INFO - -# Author: Rick Falkvinge -# -# Please remove the "Author" lines above and replace them -# with your own name if you copy and modify this script. - -# Do NOT "set -e" - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/usr/sbin:/usr/bin:/sbin:/bin -DESC="Swarmops Backend" -NAME=Swarmops.Backend.exe -MONO=/usr/bin/mono -RUNDIR=/usr/share/swarmops/backend -DAEMON=$RUNDIR/$NAME -DAEMON_ARGS="" -PIDFILE=/var/run/$NAME.pid -VARDIR=/var/run/swarmops -SCRIPTNAME=/etc/init.d/$NAME -KILLFILE=$RUNDIR/swarmops-backend-exit.flag -USRID="swarmops" - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $MONO \ - --make-pidfile --chuid $USRID --chdir $RUNDIR --background --te$ - || return 1 - start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $MONO \ - --make-pidfile --chuid $USRID --chdir $RUNDIR --background -- $$ - || return 2 - # Add code here, if necessary, that waits for the process to be ready - # to handle requests from services started subsequently which depend - # on this one. As a last resort, sleep for some time. -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - PID=$(cat $PIDFILE) - - touch $KILLFILE - for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do - if [ ! -d /proc/$PID ]; then - rm -f $PIDFILE - return 0 - fi - sleep 1 - done - - return 2 -} - -# -# Function that sends a SIGHUP to the daemon/service -# -do_reload() { - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME - return 0 -} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - status) - status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - #reload|force-reload) - # - # If do_reload() is not implemented then leave this commented out - # and leave 'force-reload' as an alias for 'restart'. - # - #log_daemon_msg "Reloading $DESC" "$NAME" - #do_reload - #log_end_msg $? - #;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac - -: \ No newline at end of file diff --git a/Backend/etcinitd/swarmops-backend b/Backend/etcinitd/swarmops-backend deleted file mode 100644 index 744734ae..00000000 --- a/Backend/etcinitd/swarmops-backend +++ /dev/null @@ -1,170 +0,0 @@ -#! /bin/sh -### BEGIN INIT INFO -# Provides: swarmops-backend -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Swarmops backend -# Short-Description: Swarmops backend -# Description: This file starts and stops the Swarmsops backend. It is installed as part of -# the swarmops-backend package and placed in /etc/init.d for compatibility with -# Debian and Ubuntu servers. -### END INIT INFO - -# Author: Rick Falkvinge -# -# Please remove the "Author" lines above and replace them -# with your own name if you copy and modify this script. - -# Do NOT "set -e" - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="Swarmops backend daemon" -NAME=swarmops-back -BINARY=Swarmops.Backend.exe -MONO=/usr/bin/mono -DAEMON=/usr/share/swarmops/backend -DAEMON_ARGS="--options args" -PIDFILE=/var/run/swarmops-backend.pid -SCRIPTNAME=/etc/init.d/swarmops-backend -LOGDIR=/var/log/swarmops - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Assert that log dir exists -if [ ! -d $LOGDIR ]; then - mkdir $LOGDIR -fi - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - echo "Starting Swarmops backend..." - start-stop-daemon --start --pidfile $PIDFILE --exec $MONO --make-pidfile --chdir $DAEMON --name $NAME --background --test $DAEMON/$BINARY > /dev/null \ - || return 1 - start-stop-daemon --start --pidfile $PIDFILE --exec $MONO --make-pidfile --chdir $DAEMON --name $NAME --background $DAEMON/$BINARY \ - || return 2 - # Add code here, if necessary, that waits for the process to be ready - # to handle requests from services started subsequently which depend - # on this one. As a last resort, sleep for some time. -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - echo "Stopping Swarmops backend..." - start-stop-daemon --stop --retry=TERM/120/KILL/5 --pidfile $PIDFILE - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Wait for children to finish too if this is a daemon that forks - # and if the daemon is only ever run from this initscript. - # If the above conditions are not satisfied then add some other code - # that waits for the process to drop all resources that could be - # needed by services started subsequently. A last resort is to - # sleep for some time. - start-stop-daemon --stop --quiet --oknodo --retry=0/120/KILL/5 --exec $DAEMON - [ "$?" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - -# -# Function that sends a SIGHUP to the daemon/service -# -do_reload() { - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME - return 0 -} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - status) - status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - #reload|force-reload) - # - # If do_reload() is not implemented then leave this commented out - # and leave 'force-reload' as an alias for 'restart'. - # - #log_daemon_msg "Reloading $DESC" "$NAME" - #do_reload - #log_end_msg $? - #;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/Backend/libsystemd/swarmops-backend.service b/Backend/libsystemd/swarmops-backend.service index 261598e1..1a63d458 100644 --- a/Backend/libsystemd/swarmops-backend.service +++ b/Backend/libsystemd/swarmops-backend.service @@ -6,7 +6,8 @@ After=network.target User=root ExecStart=/usr/bin/mono /usr/share/swarmops/backend/Swarmops.Backend.exe WorkingDirectory=/usr/share/swarmops/backend -Restart=on-abort +Restart=on-failure +RestartSec=30 [Install] diff --git a/Common/ExtensionMethods/DateTime.cs b/Common/ExtensionMethods/DateTime.cs index 8b9ffe47..d6c5afd9 100644 --- a/Common/ExtensionMethods/DateTime.cs +++ b/Common/ExtensionMethods/DateTime.cs @@ -5,11 +5,11 @@ namespace Swarmops.Common.ExtensionMethods { public static partial class DummyWrapper { - public static Int32 ToUnix (this DateTime source) + public static UInt64 ToUnix (this DateTime source) { // Assumes DateTime is already UTC - return (Int32)(source.Subtract(new DateTime(1970, 1, 1, 0, 0, 0,DateTimeKind.Utc))).TotalSeconds; + return (UInt64)(source.Subtract(new DateTime(1970, 1, 1, 0, 0, 0,DateTimeKind.Utc))).TotalSeconds; } public static bool IsHigh(this DateTime source) diff --git a/FrontSocket/Frontend-Socket.csproj b/FrontSocket/Frontend-Socket.csproj index b0580c6a..69fa01ef 100644 --- a/FrontSocket/Frontend-Socket.csproj +++ b/FrontSocket/Frontend-Socket.csproj @@ -62,7 +62,7 @@ - + diff --git a/FrontSocket/FrontendLoop.cs b/FrontSocket/FrontendLoop.cs index f983c2e4..6afd0dff 100644 --- a/FrontSocket/FrontendLoop.cs +++ b/FrontSocket/FrontendLoop.cs @@ -21,6 +21,17 @@ namespace Swarmops.Frontend.Socket { static void Main(string[] args) { + // Are we running yet? + + if (!SystemSettings.DatabaseInitialized) + { + // will restart the service every 30s until db initialized on OOBE + // also, the read of DatabaseInitialized can and will fail if + // we're not initalized enough to even have a database + + throw new InvalidOperationException(); + } + // Check if we're Sandbox if (PilotInstallationIds.IsPilot(PilotInstallationIds.DevelopmentSandbox)) @@ -163,6 +174,8 @@ namespace Swarmops.Frontend.Socket private static void OnEveryTenSeconds() { + SystemSettings.HeartbeatFrontend = DateTime.UtcNow.ToUnix(); + _socketServer.WebSocketServices.Broadcast ("{\"messageType\":\"Heartbeat\"}"); if (_isSandbox) diff --git a/FrontSocket/libsystemd/swarmops-frontend.service b/FrontSocket/libsystemd/swarmops-frontend.service index 02cbc240..7bf7251f 100644 --- a/FrontSocket/libsystemd/swarmops-frontend.service +++ b/FrontSocket/libsystemd/swarmops-frontend.service @@ -6,7 +6,8 @@ After=network.target User=root ExecStart=/usr/bin/mono /usr/share/swarmops/frontend/bin/Swarmops.Frontend.Socket.exe WorkingDirectory=/usr/share/swarmops/frontend -Restart=on-abort +Restart=on-failure +RestartSec=30 [Install] WantedBy=multi-user.target diff --git a/Logic/Support/SystemSettings.cs b/Logic/Support/SystemSettings.cs index 7cda4424..fb60d21f 100644 --- a/Logic/Support/SystemSettings.cs +++ b/Logic/Support/SystemSettings.cs @@ -222,6 +222,65 @@ namespace Swarmops.Logic.Support } + public static UInt64 HeartbeatBackend + { + get + { + string result = Persistence.Key["HeartbeatBackend"]; + if (!string.IsNullOrEmpty(result)) + { + try + { + return UInt64.Parse(result); + } + catch (Exception) + { + return 0; + } + } + + return 0; + } + set + { + Persistence.Key["HeartbeatBackend"] = value.ToString(CultureInfo.InvariantCulture); + } + } + + + + public static UInt64 HeartbeatFrontend + { + get + { + string result = Persistence.Key["HeartbeatFrontend"]; + if (!string.IsNullOrEmpty(result)) + { + try + { + return UInt64.Parse(result); + } + catch (Exception) + { + return 0; + } + } + + return 0; + } + set + { + Persistence.Key["HeartbeatFrontend"] = value.ToString(CultureInfo.InvariantCulture); + } + } + + + public static bool DatabaseInitialized + { + get { return !String.IsNullOrEmpty(InstallationId); } + } + + public static string EtcFolder { get diff --git a/Site/Pages/v5/Public/Initialize.aspx b/Site/Pages/v5/Public/Initialize.aspx index abd13242..027897ae 100644 --- a/Site/Pages/v5/Public/Initialize.aspx +++ b/Site/Pages/v5/Public/Initialize.aspx @@ -16,6 +16,7 @@ + @@ -760,9 +761,20 @@ diff --git a/Site/Pages/v5/Public/Initialize.aspx.cs b/Site/Pages/v5/Public/Initialize.aspx.cs index ee4d8a54..89b13cc0 100644 --- a/Site/Pages/v5/Public/Initialize.aspx.cs +++ b/Site/Pages/v5/Public/Initialize.aspx.cs @@ -12,7 +12,9 @@ using System.Web.Services; using System.Web.SessionState; using System.Web.UI; using Resources; +using Swarmops.Common; using Swarmops.Common.Enums; +using Swarmops.Common.ExtensionMethods; using Swarmops.Database; using Swarmops.Logic.Cache; using Swarmops.Logic.Financial; @@ -448,7 +450,6 @@ namespace Swarmops.Frontend.Pages.v5.Public DatabaseMaintenance.UpgradeSchemata(); Thread.Sleep (100); - // SECURITY: With schemata to hold them in place, initialize the encryption keys Authentication.InitializeSymmetricDatabaseKey(); @@ -457,9 +458,10 @@ namespace Swarmops.Frontend.Pages.v5.Public // Set Geography at baseline (TODO: Ask for what baseline we got) Persistence.Key["LastGeographyUpdateId"] = "0"; - Persistence.Key["LastGeographyUpdate"] = "1900-01-01"; + Persistence.Key["LastGeographyUpdate"] = Constants.DateTimeLow.ToString("yyyy-MM-dd"); // Set an installation ID + // Doubles as start signal to daemons (if installation ID exists, db is ready for processing) Persistence.Key["SwarmopsInstallationId"] = Guid.NewGuid().ToString(); @@ -642,6 +644,31 @@ namespace Swarmops.Frontend.Pages.v5.Public return false; } + public class AjaxCallDaemonResult : AjaxCallResult + { + public bool Frontend { get; set; } + public bool Backend { get; set; } + } + + [WebMethod] + public static AjaxCallDaemonResult GetDaemonStatus() + { + UInt64 unixNow = DateTime.UtcNow.ToUnix(); + + AjaxCallDaemonResult result = new AjaxCallDaemonResult(); + if (SystemSettings.HeartbeatBackend + 60 > unixNow) + { + result.Backend = true; // backend heart is beating + } + if (SystemSettings.HeartbeatFrontend + 60 > unixNow) + { + result.Frontend = true; // frontend heart is beating + } + + result.Success = true; // call as such succeeded + return result; + } + public static bool IsLocalhost() { string localAddress = HttpContext.Current.Request.ServerVariables["LOCAL_ADDR"];