From cda126018f0a921d2981eb78e0c8ebbee5688ec3 Mon Sep 17 00:00:00 2001 From: RobbStarkAustria Date: Thu, 26 Mar 2026 21:44:52 +0100 Subject: [PATCH] Add screenshot consent notice and improve video frame/audio handling - add required dashboard screenshot consent/privacy notice to README - update copilot instructions to preserve consent notice in docs - document external VLC audio behavior (`--no-audio` for muted/0%, `--gain` otherwise) - capture real video frames for dashboard screenshots instead of black compositor output - make ffmpeg fallback frame capture playback-relative (loop-aware) to avoid repeated stale image --- .github/copilot-instructions.md | 5 + README.md | 13 +++ src/display_manager.py | 197 ++++++++++++++++++++++++++------ 3 files changed, 182 insertions(+), 33 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 2662adb..5903197 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -9,6 +9,7 @@ - ✅ **Virtual environment MUST have** pygame + pillow (required for Impressive) - ✅ **Client-side resize/compress** screenshots before MQTT transmission - ✅ **Server renders PPTX → PDF via Gotenberg** (client only displays PDFs, no LibreOffice needed) +- ✅ **Keep screenshot consent notice in docs** when describing dashboard screenshot feature ### Key Files & Locations - **Display logic**: `src/display_manager.py` (controls presentations/video/web) @@ -261,6 +262,8 @@ FILE_SERVER_SCHEME=http # http or https - **Fields**: `url`, `autoplay` (bool), `loop` (bool), `volume` (0.0-1.0 → 0-100) - **URL rewriting**: `server` host → configured file server - **Fullscreen**: enforced for python-vlc on startup (with short retry toggles); external fallback uses `--fullscreen` +- **External VLC audio**: `muted=true` (or effective volume 0%) starts with `--no-audio`; otherwise startup loudness is applied via `--gain=<0.00-1.00>` +- **Runtime volume semantics**: python-vlc supports live updates; external VLC fallback is startup-parameter based - **Monitoring PID semantics**: python-vlc runs in-process, so PID is `display_manager.py` runtime PID; external fallback uses external `vlc` PID - **HW decode errors**: `h264_v4l2m2m` failures are normal if V4L2 M2M unavailable; use software decode - Robust payload parsing with fallbacks @@ -298,6 +301,8 @@ FILE_SERVER_SCHEME=http # http or https - The Display Manager now prefers using python-vlc (libvlc) when available for video playback. This enables programmatic control (autoplay, loop, volume) and cleaner termination/cleanup. If python-vlc is not available, the external `vlc` binary is used as a fallback. - Supported video event fields: `url`, `autoplay` (boolean), `loop` (boolean), `volume` (float 0.0-1.0). The manager converts `volume` to VLC's 0-100 scale. +- External VLC fallback applies audio at startup: `--no-audio` when muted/effective 0%, otherwise `--gain` from effective volume. +- Live volume adjustments are reliable in python-vlc mode; external VLC fallback uses startup parameters and should be treated as static per launch. - URLs using the placeholder host `server` (for example `http://server:8000/...`) are rewritten to the configured file server before playback. The resolution priority is: `FILE_SERVER_BASE_URL` > `FILE_SERVER_HOST` (or `MQTT_BROKER`) + `FILE_SERVER_PORT` + `FILE_SERVER_SCHEME`. - Hardware-accelerated decoding errors (e.g., `h264_v4l2m2m`) may appear when the platform does not expose a V4L2 M2M device. To avoid these errors the Display Manager can be configured to disable hw-decoding (see README env var `VLC_HW_ACCEL`). By default the manager will attempt hw-acceleration when libvlc supports it. - Fullscreen / kiosk: the manager will attempt to make libVLC windows fullscreen (remove decorations) when using python-vlc, and the README contains recommended system-level kiosk/session setup for a truly panel-free fullscreen experience. diff --git a/README.md b/README.md index a726036..409e32e 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,10 @@ Notes: - `volume` (float): 0.0–1.0 (mapped internally to VLC's 0–100 volume scale). - Effective playback volume is calculated as `event.video.volume * client_config.audio.video_volume_multiplier` and then mapped to VLC's 0–100 scale. Example: `volume: 0.8` with `audio.video_volume_multiplier: 0.5` results in 40% VLC volume. - If `python-vlc` is not installed, the Display Manager will fall back to launching the external `vlc` binary. +- External VLC audio rendering behavior: + - When `muted: true` (or effective volume resolves to 0), fallback starts VLC with `--no-audio`. + - When not muted, fallback applies startup loudness with `--gain=<0.00-1.00>` derived from effective volume. + - Runtime volume updates are best-effort in `python-vlc` mode; external VLC fallback is startup-parameter based. - HDMI-CEC remains the recommended mechanism for TV power control only. TV volume via CEC is not implemented because support is device-dependent and much less reliable than controlling VLC directly. - The client-wide multiplier is intended to be sent over the existing MQTT config topic `infoscreen/{client_id}/config` and is persisted locally in `src/config/client_settings.json` for the Display Manager. - Fullscreen behavior: @@ -720,6 +724,15 @@ CEC_POWER_OFF_WAIT=2 The system includes automatic screenshot capture for dashboard monitoring with support for both X11 and Wayland display servers. +### Consent Notice (Required) + +By enabling dashboard screenshots, operators confirm they are authorized to capture and transmit the displayed content. + +- Screenshots are sent over MQTT and can include personal data, sensitive documents, or classroom/office information shown on screen. +- Obtain required user/owner consent before enabling screenshot monitoring in production. +- Apply local policy and legal requirements (for example GDPR/DSGVO) for retention, access control, and disclosure. +- This system captures image frames only; it does not record microphone audio. + ### Architecture **Two-process design:** diff --git a/src/display_manager.py b/src/display_manager.py index 5b3ec20..abc1192 100644 --- a/src/display_manager.py +++ b/src/display_manager.py @@ -564,6 +564,7 @@ class DisplayManager: self.running = True self.client_settings_mtime: Optional[float] = None self.client_volume_multiplier = 1.0 + self._video_duration_cache: Dict[str, float] = {} # Initialize health state tracking for process monitoring self.health = ProcessHealthState() @@ -644,6 +645,9 @@ class DisplayManager: def _calculate_video_volume_pct(self, video: Dict) -> int: """Combine event volume with the client-level multiplier for VLC.""" + if video.get('muted', False): + logging.info("Video volume resolved: muted=True, effective=0%") + return 0 self._load_client_settings() event_volume = self._normalize_volume_level(video.get('volume'), default=1.0) effective_volume = event_volume * self.client_volume_multiplier @@ -1128,9 +1132,14 @@ class DisplayManager: if not vlc: # Fallback to launching external vlc binary if self._command_exists('vlc'): - if vol_pct != 100: - logging.warning( - "External VLC fallback does not support reliable per-event startup volume here; requested effective volume is %d%%", + external_muted = bool(video.get('muted', False) or vol_pct == 0) + external_gain = max(0.0, min(1.0, vol_pct / 100.0)) + if external_muted: + logging.info("External VLC fallback: forcing mute via --no-audio") + else: + logging.info( + "External VLC fallback: applying startup gain %.2f from effective volume %d%%", + external_gain, vol_pct, ) cmd = [ @@ -1140,6 +1149,11 @@ class DisplayManager: '--loop' if video.get('loop', False) else '--play-and-exit', video_url ] + if external_muted: + cmd.insert(-1, '--no-audio') + else: + # VLC CLI gain is a linear multiplier where 1.0 is default. + cmd.insert(-1, f'--gain={external_gain:.2f}') video_log_path = os.path.join(os.path.dirname(LOG_PATH), 'video_player.out.log') os.makedirs(os.path.dirname(video_log_path), exist_ok=True) video_log = open(video_log_path, 'ab', buffering=0) @@ -1167,7 +1181,13 @@ class DisplayManager: # Use libvlc via python-vlc try: # Keep python-vlc behavior aligned with external vlc fullscreen mode. - instance = vlc.Instance('--fullscreen', '--no-video-title-show') + _muted = video.get('muted', False) or vol_pct == 0 + _instance_args = ['--fullscreen', '--no-video-title-show'] + if _muted: + # Disable audio output entirely at the instance level to avoid + # the race between async audio init and audio_set_volume(0). + _instance_args.append('--no-audio') + instance = vlc.Instance(*_instance_args) def _force_fullscreen(player_obj, label: str): """Retry fullscreen toggle because video outputs may attach asynchronously.""" @@ -1189,42 +1209,29 @@ class DisplayManager: loop_flag = bool(video.get('loop', False)) if loop_flag: - # Use MediaListPlayer for looped playback - mlp = vlc.MediaListPlayer() + # Use a plain MediaPlayer with :input-repeat so everything stays on + # our custom `instance` (e.g. with --no-audio when muted). + # MediaListPlayer() would create its own default instance, + # bypassing flags like --no-audio. mp = instance.media_player_new() - mlp.set_media_player(mp) - # Create media list with the single URL - try: - ml = instance.media_list_new([video_url]) - except Exception: - # Fallback: create empty and add media - ml = instance.media_list_new() - m = instance.media_new(video_url) - ml.add_media(m) + m = instance.media_new(video_url) + m.add_option(':input-repeat=65535') # ~infinite loop + mp.set_media(m) - mlp.set_media_list(ml) - try: - # Set loop playback mode if available - mlp.set_playback_mode(vlc.PlaybackMode.loop) - except Exception: - pass - - # Set volume on underlying media player - self._apply_vlc_volume(mp, vol_pct, 'python-vlc MediaListPlayer') + self._apply_vlc_volume(mp, vol_pct, 'python-vlc loop MediaPlayer') if autoplay: try: - mlp.play() - self._apply_vlc_volume(mp, vol_pct, 'python-vlc MediaListPlayer', retries=True) - _force_fullscreen(mp, 'python-vlc MediaListPlayer') + mp.play() + self._apply_vlc_volume(mp, vol_pct, 'python-vlc loop MediaPlayer', retries=True) + _force_fullscreen(mp, 'python-vlc loop MediaPlayer') except Exception as e: - logging.error(f"Failed to play media list: {e}") + logging.error(f"Failed to start loop media player: {e}") event_id = self.get_event_identifier(event) runtime_pid = os.getpid() - logging.info(f"Video started via python-vlc (MediaListPlayer), runtime PID: {runtime_pid}") - # python-vlc runs in-process (no external vlc child), so publish this process PID. + logging.info(f"Video started via python-vlc (loop MediaPlayer), runtime PID: {runtime_pid}") self.health.update_running(event_id, 'video', 'vlc', runtime_pid) - return DisplayProcess(process=None, event_type='video', event_id=event_id, log_file=None, log_path=None, player=mlp, volume_pct=vol_pct) + return DisplayProcess(process=None, event_type='video', event_id=event_id, log_file=None, log_path=None, player=mp, volume_pct=vol_pct) else: # Single-play MediaPlayer @@ -1766,9 +1773,14 @@ class DisplayManager: display_env = os.environ.get('DISPLAY') logging.debug(f"Screenshot session={session} DISPLAY={display_env} WAYLAND_DISPLAY={os.environ.get('WAYLAND_DISPLAY')}") + # Video planes can be black in compositor screenshots; try direct frame extraction first. + if self._capture_video_frame(raw_path): + captured = True + logging.debug("Screenshot source: direct video frame extraction") + if session == 'wayland': # 1W: grim (wayland/sway, wlroots) captures root - if command_exists('grim'): + if not captured and command_exists('grim'): cmd = ['grim', raw_path] result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15) if result.returncode == 0 and os.path.exists(raw_path): @@ -1794,7 +1806,7 @@ class DisplayManager: else: # X11 path # 1X: scrot - if command_exists('scrot'): + if not captured and command_exists('scrot'): cmd = ['scrot', raw_path] result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15) if result.returncode == 0 and os.path.exists(raw_path): @@ -1886,6 +1898,125 @@ class DisplayManager: except Exception as e: logging.debug(f"Screenshot capture failure: {e}") + def _capture_video_frame(self, target_path: str) -> bool: + """Capture a frame from the active video event directly. + + Returns True when a frame was successfully written to target_path. + """ + try: + if not self.current_process or self.current_process.event_type != 'video': + return False + + # Prefer python-vlc direct snapshot when available. + player = getattr(self.current_process, 'player', None) + if player: + media_player = None + if hasattr(player, 'video_take_snapshot'): + media_player = player + elif hasattr(player, 'get_media_player'): + try: + media_player = player.get_media_player() + except Exception: + media_player = None + + if media_player and hasattr(media_player, 'video_take_snapshot'): + try: + rc = media_player.video_take_snapshot(0, target_path, 0, 0) + if rc == 0 and os.path.exists(target_path) and os.path.getsize(target_path) > 0: + return True + logging.debug(f"python-vlc snapshot failed with rc={rc}") + except Exception as e: + logging.debug(f"python-vlc snapshot error: {e}") + + # Fallback: extract one frame from source URL using ffmpeg. + # This covers external VLC fallback mode where no python-vlc player object exists. + if not self._command_exists('ffmpeg'): + return False + + video = self.current_event_data.get('video', {}) if isinstance(self.current_event_data, dict) else {} + source_url = video.get('url') if isinstance(video, dict) else None + source_url = self._resolve_file_url(source_url) if source_url else None + if not source_url: + return False + + loop_enabled = bool(video.get('loop', False)) if isinstance(video, dict) else False + elapsed_seconds = 0.0 + try: + elapsed_seconds = max( + 0.0, + (datetime.now(timezone.utc) - self.current_process.start_time).total_seconds() + ) + except Exception: + elapsed_seconds = 0.0 + + # Use a playback-relative seek point so repeated captures are not the same frame. + seek_seconds = max(0.2, elapsed_seconds) + + duration_seconds = self._get_video_duration_seconds(source_url) + if duration_seconds and duration_seconds > 1.0: + safe_span = max(0.5, duration_seconds - 0.5) + if loop_enabled: + seek_seconds = 0.25 + (elapsed_seconds % safe_span) + else: + seek_seconds = min(seek_seconds, safe_span) + + cmd = [ + 'ffmpeg', + '-y', + '-hide_banner', + '-loglevel', 'error', + '-ss', f'{seek_seconds:.3f}', + '-i', source_url, + '-frames:v', '1', + target_path, + ] + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20) + if result.returncode == 0 and os.path.exists(target_path) and os.path.getsize(target_path) > 0: + logging.debug( + "Video frame extracted via ffmpeg: seek=%.3fs elapsed=%.3fs duration=%s loop=%s", + seek_seconds, + elapsed_seconds, + f"{duration_seconds:.3f}s" if duration_seconds else "unknown", + loop_enabled, + ) + return True + + logging.debug(f"ffmpeg frame capture failed rc={result.returncode}: {result.stderr.decode(errors='ignore')[:160]}") + return False + except Exception as e: + logging.debug(f"Direct video frame capture failed: {e}") + return False + + def _get_video_duration_seconds(self, source_url: str) -> Optional[float]: + """Return media duration in seconds for a video URL, using a small in-memory cache.""" + try: + cached = self._video_duration_cache.get(source_url) + if cached and cached > 0: + return cached + + if not self._command_exists('ffprobe'): + return None + + cmd = [ + 'ffprobe', + '-v', 'error', + '-show_entries', 'format=duration', + '-of', 'default=noprint_wrappers=1:nokey=1', + source_url, + ] + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10) + if result.returncode != 0: + return None + + raw = result.stdout.decode('utf-8', errors='ignore').strip() + duration = float(raw) + if duration > 0: + self._video_duration_cache[source_url] = duration + return duration + return None + except Exception: + return None + def main(): """Entry point"""