feat: remote commands, systemd units, process observability, broker auth split
- Command intake (reboot/shutdown) on infoscreen/{uuid}/commands with ack lifecycle
- MQTT_USER/MQTT_PASSWORD_BROKER split from identity vars; configure_mqtt_security() updated
- infoscreen-simclient.service: Type=notify, WatchdogSec=60, Restart=on-failure
- infoscreen-notify-failure@.service + script: retained MQTT alert when systemd gives up (Gap 3)
- _sd_notify() watchdog keepalive in simclient main loop (Gap 1)
- broker_connection block in health payload: reconnect_count, last_disconnect_at (Gap 2)
- COMMAND_MOCK_REBOOT_IMMEDIATE_COMPLETE canary flag with safety guard
- SERVER_TEAM_ACTIONS.md: server-side integration action items
- Docs: README, CHANGELOG, src/README, copilot-instructions updated
- 43 tests passing
This commit is contained in:
RobbStarkAustria
27
scripts/infoscreen-cmd-helper.sh
Executable file
27
scripts/infoscreen-cmd-helper.sh
Executable file
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Privileged command helper for remote reboot/shutdown actions.
|
||||
# Intended installation path: /usr/local/bin/infoscreen-cmd-helper.sh
|
||||
# Suggested sudoers entry:
|
||||
# infoscreen ALL=(ALL) NOPASSWD: /usr/local/bin/infoscreen-cmd-helper.sh
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
echo "usage: infoscreen-cmd-helper.sh <reboot_host|shutdown_host>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
action="$1"
|
||||
|
||||
case "$action" in
|
||||
reboot_host)
|
||||
exec systemctl reboot
|
||||
;;
|
||||
shutdown_host)
|
||||
exec systemctl poweroff
|
||||
;;
|
||||
*)
|
||||
echo "unsupported action: $action" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -3,6 +3,8 @@ Description=Infoscreen Display Manager
|
||||
Documentation=https://github.com/RobbStarkAustria/infoscreen_client_2025
|
||||
After=network.target graphical.target
|
||||
Wants=network-online.target
|
||||
# Publish an MQTT alert if systemd gives up restarting (StartLimitBurst exceeded).
|
||||
OnFailure=infoscreen-notify-failure@%n.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
55
scripts/infoscreen-notify-failure.sh
Executable file
55
scripts/infoscreen-notify-failure.sh
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# Publishes a service-failed MQTT notification when called by systemd OnFailure=.
|
||||
# Usage: infoscreen-notify-failure.sh <failing-unit-name>
|
||||
#
|
||||
# Designed to be called from infoscreen-notify-failure@.service.
|
||||
# Reads broker credentials from .env; reads client UUID from config.
|
||||
# Safe to run even if MQTT is unreachable (exits cleanly, errors logged to journal).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
FAILING_UNIT="${1:-unknown}"
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
ENV_FILE="$PROJECT_DIR/.env"
|
||||
UUID_FILE="$PROJECT_DIR/src/config/client_uuid.txt"
|
||||
|
||||
# Load .env (skip comments and blank lines)
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source <(grep -v '^\s*#' "$ENV_FILE" | grep -v '^\s*$')
|
||||
set +a
|
||||
fi
|
||||
|
||||
MQTT_BROKER="${MQTT_BROKER:-localhost}"
|
||||
MQTT_PORT="${MQTT_PORT:-1883}"
|
||||
MQTT_USER="${MQTT_USER:-}"
|
||||
MQTT_PASSWORD_BROKER="${MQTT_PASSWORD_BROKER:-}"
|
||||
|
||||
CLIENT_UUID="unknown"
|
||||
if [[ -f "$UUID_FILE" ]]; then
|
||||
CLIENT_UUID="$(cat "$UUID_FILE" | tr -d '[:space:]')"
|
||||
fi
|
||||
|
||||
TOPIC="infoscreen/${CLIENT_UUID}/service_failed"
|
||||
TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
PAYLOAD=$(printf '{"event":"service_failed","unit":"%s","client_uuid":"%s","failed_at":"%s"}' \
|
||||
"$FAILING_UNIT" "$CLIENT_UUID" "$TIMESTAMP")
|
||||
|
||||
# Build mosquitto_pub auth args
|
||||
AUTH_ARGS=()
|
||||
if [[ -n "$MQTT_USER" ]]; then AUTH_ARGS+=(-u "$MQTT_USER"); fi
|
||||
if [[ -n "$MQTT_PASSWORD_BROKER" ]]; then AUTH_ARGS+=(-P "$MQTT_PASSWORD_BROKER"); fi
|
||||
|
||||
echo "Publishing service-failed notification: unit=$FAILING_UNIT client=$CLIENT_UUID"
|
||||
|
||||
mosquitto_pub \
|
||||
-h "$MQTT_BROKER" \
|
||||
-p "$MQTT_PORT" \
|
||||
"${AUTH_ARGS[@]}" \
|
||||
-t "$TOPIC" \
|
||||
-m "$PAYLOAD" \
|
||||
-q 1 \
|
||||
--retain \
|
||||
2>&1 || echo "WARNING: mosquitto_pub failed (broker unreachable?); notification not delivered"
|
||||
19
scripts/infoscreen-notify-failure@.service
Normal file
19
scripts/infoscreen-notify-failure@.service
Normal file
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Infoscreen service-failed MQTT notifier (%i)
|
||||
# One-shot: run once and exit. %i is the failing unit name passed by OnFailure=.
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=olafn
|
||||
Group=olafn
|
||||
WorkingDirectory=/home/olafn/infoscreen-dev
|
||||
EnvironmentFile=/home/olafn/infoscreen-dev/.env
|
||||
ExecStart=/home/olafn/infoscreen-dev/scripts/infoscreen-notify-failure.sh %i
|
||||
|
||||
# Do not restart the notifier itself.
|
||||
Restart=no
|
||||
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=infoscreen-notify-failure
|
||||
55
scripts/infoscreen-simclient.service
Normal file
55
scripts/infoscreen-simclient.service
Normal file
@@ -0,0 +1,55 @@
|
||||
[Unit]
|
||||
Description=Infoscreen Simclient (MQTT communication)
|
||||
Documentation=https://github.com/RobbStarkAustria/infoscreen_client_2025
|
||||
# Simclient needs network before starting — MQTT will fail otherwise.
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Publish an MQTT alert if systemd gives up restarting (StartLimitBurst exceeded).
|
||||
OnFailure=infoscreen-notify-failure@%n.service
|
||||
|
||||
[Service]
|
||||
# notify: simclient sends READY=1 via sd_notify once fully initialised.
|
||||
# WatchdogSec: if WATCHDOG=1 is not sent within this window, systemd kills
|
||||
# and restarts the process — detects hung/deadlocked main loops.
|
||||
Type=notify
|
||||
WatchdogSec=60
|
||||
User=olafn
|
||||
Group=olafn
|
||||
WorkingDirectory=/home/olafn/infoscreen-dev
|
||||
|
||||
# Load all client configuration from the local .env file.
|
||||
# Keep .env mode 600; systemd reads it as root before dropping privileges.
|
||||
EnvironmentFile=/home/olafn/infoscreen-dev/.env
|
||||
|
||||
# Start simclient
|
||||
ExecStart=/home/olafn/infoscreen-dev/scripts/start-simclient.sh
|
||||
|
||||
# Restart on failure (non-zero exit or signal).
|
||||
# This covers crash recovery AND the reboot-command lifecycle:
|
||||
# 1. Server sends reboot_host command
|
||||
# 2. Simclient publishes accepted + execution_started, then exits
|
||||
# 3. Systemd restarts simclient within RestartSec seconds
|
||||
# 4. On reconnect, heartbeat loop detects pending_recovery_command and
|
||||
# publishes completed — closing the lifecycle cleanly.
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
# Prevent rapid restart thrash: allow at most 5 restarts in 60 seconds.
|
||||
StartLimitIntervalSec=60
|
||||
StartLimitBurst=5
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=infoscreen-simclient
|
||||
|
||||
# Security settings
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
|
||||
# Resource limits
|
||||
LimitNOFILE=65536
|
||||
|
||||
[Install]
|
||||
# Simclient runs in multi-user mode — no graphical session required.
|
||||
WantedBy=multi-user.target
|
||||
24
scripts/install-command-helper.sh
Executable file
24
scripts/install-command-helper.sh
Executable file
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Installs the privileged command helper and sudoers drop-in.
|
||||
# Usage: ./scripts/install-command-helper.sh [linux-user]
|
||||
|
||||
target_user="${1:-$USER}"
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
helper_src="$script_dir/infoscreen-cmd-helper.sh"
|
||||
helper_dst="/usr/local/bin/infoscreen-cmd-helper.sh"
|
||||
sudoers_file="/etc/sudoers.d/infoscreen-command-helper"
|
||||
|
||||
if [[ ! -f "$helper_src" ]]; then
|
||||
echo "helper source not found: $helper_src" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo install -m 0755 "$helper_src" "$helper_dst"
|
||||
printf '%s\n' "$target_user ALL=(ALL) NOPASSWD: $helper_dst" | sudo tee "$sudoers_file" >/dev/null
|
||||
sudo chmod 0440 "$sudoers_file"
|
||||
sudo visudo -cf "$sudoers_file" >/dev/null
|
||||
|
||||
echo "Installed helper: $helper_dst"
|
||||
echo "Installed sudoers: $sudoers_file (user: $target_user)"
|
||||
34
scripts/mock-command-helper.sh
Executable file
34
scripts/mock-command-helper.sh
Executable file
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Non-destructive helper for command lifecycle canary tests.
|
||||
# Use by starting simclient with:
|
||||
# COMMAND_HELPER_PATH=/home/olafn/infoscreen-dev/scripts/mock-command-helper.sh
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
echo "usage: mock-command-helper.sh <reboot_host|shutdown_host>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
action="$1"
|
||||
|
||||
case "$action" in
|
||||
reboot_host|shutdown_host)
|
||||
;;
|
||||
*)
|
||||
echo "unsupported action: $action" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "${MOCK_COMMAND_HELPER_FORCE_FAIL:-0}" == "1" ]]; then
|
||||
echo "forced failure for canary test (action=$action)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${MOCK_COMMAND_HELPER_SLEEP_SEC:-0}" != "0" ]]; then
|
||||
sleep "${MOCK_COMMAND_HELPER_SLEEP_SEC}"
|
||||
fi
|
||||
|
||||
echo "mock helper executed action=$action"
|
||||
exit 0
|
||||
35
scripts/start-simclient.sh
Executable file
35
scripts/start-simclient.sh
Executable file
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
# Start Simclient - MQTT communication and event intake for infoscreen
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
VENV_PATH="$PROJECT_ROOT/venv"
|
||||
SIMCLIENT="$PROJECT_ROOT/src/simclient.py"
|
||||
|
||||
echo "📡 Starting Simclient..."
|
||||
echo "Project root: $PROJECT_ROOT"
|
||||
|
||||
# Check if virtual environment exists
|
||||
if [ ! -d "$VENV_PATH" ]; then
|
||||
echo "❌ Virtual environment not found at: $VENV_PATH"
|
||||
echo "Please create it with: python3 -m venv venv"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Activate virtual environment
|
||||
source "$VENV_PATH/bin/activate"
|
||||
|
||||
# Check if simclient.py exists
|
||||
if [ ! -f "$SIMCLIENT" ]; then
|
||||
echo "❌ Simclient not found at: $SIMCLIENT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ENV="${ENV:-development}"
|
||||
echo "Environment: $ENV"
|
||||
|
||||
echo "Starting simclient..."
|
||||
echo "---"
|
||||
exec python3 "$SIMCLIENT"
|
||||
@@ -1,9 +1,27 @@
|
||||
#!/bin/bash
|
||||
source "$(dirname "$0")/../.env"
|
||||
|
||||
MQTT_AUTH_ARGS=()
|
||||
MQTT_TLS_ARGS=()
|
||||
|
||||
if [[ -n "${MQTT_USERNAME:-}" ]]; then
|
||||
MQTT_AUTH_ARGS+=( -u "$MQTT_USERNAME" )
|
||||
fi
|
||||
if [[ -n "${MQTT_PASSWORD:-}" ]]; then
|
||||
MQTT_AUTH_ARGS+=( -P "$MQTT_PASSWORD" )
|
||||
fi
|
||||
if [[ "${MQTT_TLS_ENABLED:-0}" == "1" || "${MQTT_TLS_ENABLED:-0}" == "true" || "${MQTT_TLS_ENABLED:-0}" == "yes" ]]; then
|
||||
[[ -n "${MQTT_TLS_CA_CERT:-}" ]] && MQTT_TLS_ARGS+=( --cafile "$MQTT_TLS_CA_CERT" )
|
||||
[[ -n "${MQTT_TLS_CERT:-}" ]] && MQTT_TLS_ARGS+=( --cert "$MQTT_TLS_CERT" )
|
||||
[[ -n "${MQTT_TLS_KEY:-}" ]] && MQTT_TLS_ARGS+=( --key "$MQTT_TLS_KEY" )
|
||||
if [[ "${MQTT_TLS_INSECURE:-0}" == "1" || "${MQTT_TLS_INSECURE:-0}" == "true" || "${MQTT_TLS_INSECURE:-0}" == "yes" ]]; then
|
||||
MQTT_TLS_ARGS+=( --insecure )
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Testing MQTT connection to $MQTT_BROKER:$MQTT_PORT"
|
||||
echo "Publishing test message..."
|
||||
mosquitto_pub -h "$MQTT_BROKER" -p "$MQTT_PORT" -t "infoscreen/test" -m "Hello from Pi development setup"
|
||||
mosquitto_pub -h "$MQTT_BROKER" -p "$MQTT_PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -t "infoscreen/test" -m "Hello from Pi development setup"
|
||||
|
||||
echo "Subscribing to test topic (press Ctrl+C to stop)..."
|
||||
mosquitto_sub -h "$MQTT_BROKER" -p "$MQTT_PORT" -t "infoscreen/test"
|
||||
mosquitto_sub -h "$MQTT_BROKER" -p "$MQTT_PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -t "infoscreen/test"
|
||||
|
||||
@@ -36,6 +36,31 @@ fi
|
||||
|
||||
BROKER="${MQTT_BROKER:-localhost}"
|
||||
PORT="${MQTT_PORT:-1883}"
|
||||
MQTT_USERNAME="${MQTT_USERNAME:-}"
|
||||
MQTT_PASSWORD="${MQTT_PASSWORD:-}"
|
||||
MQTT_TLS_ENABLED="${MQTT_TLS_ENABLED:-0}"
|
||||
MQTT_TLS_CA_CERT="${MQTT_TLS_CA_CERT:-}"
|
||||
MQTT_TLS_CERT="${MQTT_TLS_CERT:-}"
|
||||
MQTT_TLS_KEY="${MQTT_TLS_KEY:-}"
|
||||
MQTT_TLS_INSECURE="${MQTT_TLS_INSECURE:-0}"
|
||||
|
||||
MQTT_AUTH_ARGS=()
|
||||
MQTT_TLS_ARGS=()
|
||||
|
||||
if [[ -n "$MQTT_USERNAME" ]]; then
|
||||
MQTT_AUTH_ARGS+=( -u "$MQTT_USERNAME" )
|
||||
fi
|
||||
if [[ -n "$MQTT_PASSWORD" ]]; then
|
||||
MQTT_AUTH_ARGS+=( -P "$MQTT_PASSWORD" )
|
||||
fi
|
||||
if [[ "$MQTT_TLS_ENABLED" == "1" || "$MQTT_TLS_ENABLED" == "true" || "$MQTT_TLS_ENABLED" == "yes" ]]; then
|
||||
[[ -n "$MQTT_TLS_CA_CERT" ]] && MQTT_TLS_ARGS+=( --cafile "$MQTT_TLS_CA_CERT" )
|
||||
[[ -n "$MQTT_TLS_CERT" ]] && MQTT_TLS_ARGS+=( --cert "$MQTT_TLS_CERT" )
|
||||
[[ -n "$MQTT_TLS_KEY" ]] && MQTT_TLS_ARGS+=( --key "$MQTT_TLS_KEY" )
|
||||
if [[ "$MQTT_TLS_INSECURE" == "1" || "$MQTT_TLS_INSECURE" == "true" || "$MQTT_TLS_INSECURE" == "yes" ]]; then
|
||||
MQTT_TLS_ARGS+=( --insecure )
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Read runtime IDs ─────────────────────────────────────────────────────────
|
||||
GROUP_ID_FILE="$PROJECT_ROOT/src/config/last_group_id.txt"
|
||||
@@ -107,7 +132,7 @@ EOF
|
||||
echo -e "${YELLOW}Publishing to: $topic${NC}"
|
||||
echo "$payload" | python3 -m json.tool 2>/dev/null || echo "$payload"
|
||||
echo ""
|
||||
mosquitto_pub -h "$BROKER" -p "$PORT" -t "$topic" -q 1 --retain -m "$payload"
|
||||
mosquitto_pub -h "$BROKER" -p "$PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -t "$topic" -q 1 --retain -m "$payload"
|
||||
echo -e "${GREEN}Published (retained, QoS 1)${NC}"
|
||||
echo "intent_id: $intent_id"
|
||||
}
|
||||
@@ -119,7 +144,7 @@ clear_intent() {
|
||||
echo -e "${RED}No group_id found.${NC}"
|
||||
return 1
|
||||
fi
|
||||
mosquitto_pub -h "$BROKER" -p "$PORT" -t "$topic" -q 1 --retain --null-message
|
||||
mosquitto_pub -h "$BROKER" -p "$PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -t "$topic" -q 1 --retain --null-message
|
||||
echo -e "${GREEN}Retained intent cleared from broker${NC}"
|
||||
}
|
||||
|
||||
@@ -167,7 +192,7 @@ subscribe_power_state() {
|
||||
echo -e "${YELLOW}Subscribing to: $topic${NC}"
|
||||
echo "(Ctrl-C to stop)"
|
||||
echo ""
|
||||
mosquitto_sub -h "$BROKER" -p "$PORT" -t "$topic" | \
|
||||
mosquitto_sub -h "$BROKER" -p "$PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -t "$topic" | \
|
||||
python3 -c "
|
||||
import sys, json
|
||||
for line in sys.stdin:
|
||||
@@ -216,7 +241,7 @@ while true; do
|
||||
echo -e "${RED}No group_id.${NC}"
|
||||
else
|
||||
TOPIC="$(group_topic)"
|
||||
mosquitto_pub -h "$BROKER" -p "$PORT" -t "$TOPIC" -q 1 --retain \
|
||||
mosquitto_pub -h "$BROKER" -p "$PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -t "$TOPIC" -q 1 --retain \
|
||||
-m '{"schema_version":"1.0","desired_state":"on"}'
|
||||
echo -e "${YELLOW}⚠ Malformed intent published - client must reject with 'missing required field' in log${NC}"
|
||||
fi
|
||||
|
||||
244
scripts/test-reboot-command.sh
Executable file
244
scripts/test-reboot-command.sh
Executable file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env bash
|
||||
# Safe end-to-end command lifecycle canary for reboot/shutdown contract v1.
|
||||
# Verifies ack flow: accepted -> execution_started -> completed/failed.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
BLUE='\033[0;34m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m'
|
||||
|
||||
ENV_FILE="$PROJECT_ROOT/.env"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
while IFS='=' read -r key value; do
|
||||
key="${key//[$'\t\r\n']}"
|
||||
key="$(echo "$key" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
[[ -z "$key" ]] && continue
|
||||
[[ "$key" =~ ^# ]] && continue
|
||||
[[ "$key" =~ ^[A-Z_][A-Z0-9_]*$ ]] || continue
|
||||
value="${value%%#*}"
|
||||
value="${value//[$'\t\r\n']}"
|
||||
value="$(echo "$value" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
export "$key=$value"
|
||||
done < "$ENV_FILE"
|
||||
fi
|
||||
|
||||
BROKER="${MQTT_BROKER:-localhost}"
|
||||
PORT="${MQTT_PORT:-1883}"
|
||||
MQTT_USERNAME="${MQTT_USERNAME:-}"
|
||||
MQTT_PASSWORD="${MQTT_PASSWORD:-}"
|
||||
MQTT_TLS_ENABLED="${MQTT_TLS_ENABLED:-0}"
|
||||
MQTT_TLS_CA_CERT="${MQTT_TLS_CA_CERT:-}"
|
||||
MQTT_TLS_CERT="${MQTT_TLS_CERT:-}"
|
||||
MQTT_TLS_KEY="${MQTT_TLS_KEY:-}"
|
||||
MQTT_TLS_INSECURE="${MQTT_TLS_INSECURE:-0}"
|
||||
CLIENT_UUID_FILE="$PROJECT_ROOT/src/config/client_uuid.txt"
|
||||
LAST_COMMAND_STATE_FILE="$PROJECT_ROOT/src/config/last_command_state.json"
|
||||
|
||||
if [[ ! -f "$CLIENT_UUID_FILE" ]]; then
|
||||
echo -e "${RED}client UUID file missing: $CLIENT_UUID_FILE${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v mosquitto_pub >/dev/null 2>&1 || ! command -v mosquitto_sub >/dev/null 2>&1; then
|
||||
echo -e "${RED}mosquitto_pub/sub not found. Install mosquitto-clients.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CLIENT_UUID="$(tr -d '[:space:]' < "$CLIENT_UUID_FILE")"
|
||||
COMMAND_TOPIC="infoscreen/${CLIENT_UUID}/commands"
|
||||
COMMAND_TOPIC_ALIAS="infoscreen/${CLIENT_UUID}/command"
|
||||
ACK_TOPIC="infoscreen/${CLIENT_UUID}/commands/ack"
|
||||
ACK_TOPIC_ALIAS="infoscreen/${CLIENT_UUID}/command/ack"
|
||||
|
||||
MQTT_AUTH_ARGS=()
|
||||
MQTT_TLS_ARGS=()
|
||||
|
||||
if [[ -n "$MQTT_USERNAME" ]]; then
|
||||
MQTT_AUTH_ARGS+=( -u "$MQTT_USERNAME" )
|
||||
fi
|
||||
if [[ -n "$MQTT_PASSWORD" ]]; then
|
||||
MQTT_AUTH_ARGS+=( -P "$MQTT_PASSWORD" )
|
||||
fi
|
||||
if [[ "$MQTT_TLS_ENABLED" == "1" || "$MQTT_TLS_ENABLED" == "true" || "$MQTT_TLS_ENABLED" == "yes" ]]; then
|
||||
[[ -n "$MQTT_TLS_CA_CERT" ]] && MQTT_TLS_ARGS+=( --cafile "$MQTT_TLS_CA_CERT" )
|
||||
[[ -n "$MQTT_TLS_CERT" ]] && MQTT_TLS_ARGS+=( --cert "$MQTT_TLS_CERT" )
|
||||
[[ -n "$MQTT_TLS_KEY" ]] && MQTT_TLS_ARGS+=( --key "$MQTT_TLS_KEY" )
|
||||
if [[ "$MQTT_TLS_INSECURE" == "1" || "$MQTT_TLS_INSECURE" == "true" || "$MQTT_TLS_INSECURE" == "yes" ]]; then
|
||||
MQTT_TLS_ARGS+=( --insecure )
|
||||
fi
|
||||
fi
|
||||
|
||||
ACTION="${1:-reboot_host}"
|
||||
MODE="${2:-success}" # success | failed
|
||||
TOPIC_MODE="${3:-canonical}" # canonical | alias
|
||||
WAIT_SEC="${4:-25}"
|
||||
|
||||
if [[ "$ACTION" != "reboot_host" && "$ACTION" != "shutdown_host" ]]; then
|
||||
echo -e "${RED}invalid action '$ACTION' (expected reboot_host|shutdown_host)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$MODE" != "success" && "$MODE" != "failed" ]]; then
|
||||
echo -e "${RED}invalid mode '$MODE' (expected success|failed)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$TOPIC_MODE" != "canonical" && "$TOPIC_MODE" != "alias" ]]; then
|
||||
echo -e "${RED}invalid topic mode '$TOPIC_MODE' (expected canonical|alias)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$WAIT_SEC" =~ ^[0-9]+$ ]] || [[ "$WAIT_SEC" -lt 1 ]]; then
|
||||
echo -e "${RED}invalid wait seconds '$WAIT_SEC' (expected positive integer)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$TOPIC_MODE" == "alias" ]]; then
|
||||
COMMAND_TOPIC="$COMMAND_TOPIC_ALIAS"
|
||||
fi
|
||||
|
||||
COMMAND_ID="$(python3 - <<'PY'
|
||||
import uuid
|
||||
print(uuid.uuid4())
|
||||
PY
|
||||
)"
|
||||
ISSUED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
EXPIRES_EPOCH="$(( $(date +%s) + 240 ))"
|
||||
EXPIRES_AT="$(date -u -d "@$EXPIRES_EPOCH" +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
|
||||
PAYLOAD="$(cat <<EOF
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"command_id": "$COMMAND_ID",
|
||||
"client_uuid": "$CLIENT_UUID",
|
||||
"action": "$ACTION",
|
||||
"issued_at": "$ISSUED_AT",
|
||||
"expires_at": "$EXPIRES_AT",
|
||||
"requested_by": 1,
|
||||
"reason": "canary_test"
|
||||
}
|
||||
EOF
|
||||
)"
|
||||
|
||||
TMP_ACK_LOG="$(mktemp)"
|
||||
cleanup() {
|
||||
[[ -n "${SUB_PID_1:-}" ]] && kill "$SUB_PID_1" >/dev/null 2>&1 || true
|
||||
[[ -n "${SUB_PID_2:-}" ]] && kill "$SUB_PID_2" >/dev/null 2>&1 || true
|
||||
rm -f "$TMP_ACK_LOG"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo -e "${BLUE}================================================${NC}"
|
||||
echo -e "${BLUE}Command Lifecycle Canary${NC}"
|
||||
echo -e "${BLUE}================================================${NC}"
|
||||
echo " Broker : $BROKER:$PORT"
|
||||
echo " Client UUID : $CLIENT_UUID"
|
||||
echo " Command ID : $COMMAND_ID"
|
||||
echo " Action : $ACTION"
|
||||
echo " Mode : $MODE"
|
||||
echo " Cmd Topic : $COMMAND_TOPIC"
|
||||
echo " Ack Topics : $ACK_TOPIC , $ACK_TOPIC_ALIAS"
|
||||
echo ""
|
||||
echo -e "${YELLOW}IMPORTANT${NC}: to avoid real reboot/shutdown, run simclient with"
|
||||
echo " COMMAND_HELPER_PATH=$PROJECT_ROOT/scripts/mock-command-helper.sh"
|
||||
echo ""
|
||||
|
||||
# Subscribe first to avoid missing retained/non-retained race windows.
|
||||
mosquitto_sub -h "$BROKER" -p "$PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -q 1 -v -t "$ACK_TOPIC" >> "$TMP_ACK_LOG" &
|
||||
SUB_PID_1=$!
|
||||
mosquitto_sub -h "$BROKER" -p "$PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -q 1 -v -t "$ACK_TOPIC_ALIAS" >> "$TMP_ACK_LOG" &
|
||||
SUB_PID_2=$!
|
||||
sleep 0.5
|
||||
|
||||
if [[ "$MODE" == "failed" ]]; then
|
||||
echo -e "${YELLOW}If simclient was started with MOCK_COMMAND_HELPER_FORCE_FAIL=1, expected terminal status is failed.${NC}"
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}Publishing command payload...${NC}"
|
||||
mosquitto_pub -h "$BROKER" -p "$PORT" "${MQTT_AUTH_ARGS[@]}" "${MQTT_TLS_ARGS[@]}" -q 1 -t "$COMMAND_TOPIC" -m "$PAYLOAD"
|
||||
|
||||
EXPECTED_TERMINAL="completed"
|
||||
if [[ "$MODE" == "failed" ]]; then
|
||||
EXPECTED_TERMINAL="failed"
|
||||
fi
|
||||
|
||||
EXPECT_RECOVERY_COMPLETION=0
|
||||
if [[ "$ACTION" == "reboot_host" && "$MODE" == "success" ]]; then
|
||||
EXPECTED_TERMINAL=""
|
||||
EXPECT_RECOVERY_COMPLETION=1
|
||||
fi
|
||||
|
||||
DEADLINE=$(( $(date +%s) + WAIT_SEC ))
|
||||
SEEN_ACCEPTED=0
|
||||
SEEN_STARTED=0
|
||||
SEEN_TERMINAL=0
|
||||
|
||||
while [[ $(date +%s) -lt $DEADLINE ]]; do
|
||||
if grep -q '"command_id"' "$TMP_ACK_LOG" 2>/dev/null; then
|
||||
if grep -q "\"command_id\": \"$COMMAND_ID\"" "$TMP_ACK_LOG"; then
|
||||
grep -q '"status": "accepted"' "$TMP_ACK_LOG" && SEEN_ACCEPTED=1 || true
|
||||
grep -q '"status": "execution_started"' "$TMP_ACK_LOG" && SEEN_STARTED=1 || true
|
||||
if [[ -n "$EXPECTED_TERMINAL" ]]; then
|
||||
grep -q "\"status\": \"$EXPECTED_TERMINAL\"" "$TMP_ACK_LOG" && SEEN_TERMINAL=1 || true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $SEEN_ACCEPTED -eq 1 && $SEEN_STARTED -eq 1 ]]; then
|
||||
if [[ -z "$EXPECTED_TERMINAL" || $SEEN_TERMINAL -eq 1 ]]; then
|
||||
break
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}Ack stream (filtered by command_id):${NC}"
|
||||
python3 - <<'PY' "$TMP_ACK_LOG" "$COMMAND_ID"
|
||||
import json
|
||||
import sys
|
||||
|
||||
path, command_id = sys.argv[1], sys.argv[2]
|
||||
with open(path, "r", encoding="utf-8", errors="ignore") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split(" ", 1)
|
||||
payload = parts[1] if len(parts) == 2 else parts[0]
|
||||
try:
|
||||
obj = json.loads(payload)
|
||||
except Exception:
|
||||
continue
|
||||
if obj.get("command_id") == command_id:
|
||||
print(json.dumps(obj, indent=2))
|
||||
PY
|
||||
|
||||
if [[ $SEEN_ACCEPTED -eq 1 && $SEEN_STARTED -eq 1 ]]; then
|
||||
if [[ -z "$EXPECTED_TERMINAL" ]]; then
|
||||
echo -e "${GREEN}PASS${NC}: observed accepted -> execution_started"
|
||||
echo -e "${YELLOW}NOTE${NC}: completed for reboot_host is expected only after client reconnect/recovery."
|
||||
elif [[ $SEEN_TERMINAL -eq 1 ]]; then
|
||||
echo -e "${GREEN}PASS${NC}: observed accepted -> execution_started -> $EXPECTED_TERMINAL"
|
||||
else
|
||||
echo -e "${RED}FAIL${NC}: missing expected terminal state $EXPECTED_TERMINAL for command_id=$COMMAND_ID"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}FAIL${NC}: missing expected lifecycle states for command_id=$COMMAND_ID"
|
||||
if [[ -n "$EXPECTED_TERMINAL" ]]; then
|
||||
echo " observed: accepted=$SEEN_ACCEPTED execution_started=$SEEN_STARTED terminal($EXPECTED_TERMINAL)=$SEEN_TERMINAL"
|
||||
else
|
||||
echo " observed: accepted=$SEEN_ACCEPTED execution_started=$SEEN_STARTED"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -f "$LAST_COMMAND_STATE_FILE" ]]; then
|
||||
echo ""
|
||||
echo -e "${BLUE}Last command state:${NC}"
|
||||
python3 -m json.tool "$LAST_COMMAND_STATE_FILE" || cat "$LAST_COMMAND_STATE_FILE"
|
||||
fi
|
||||
Reference in New Issue
Block a user