commit 7b826e9f384a96d0e0b9070a36bbaa6f2f1919fc Author: Maciej Bator Date: Wed Jan 21 14:44:17 2026 +0100 initial commit - vibe coded with claude diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a8cb1f9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +FROM python:3.11-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + ffmpeg \ + wget \ + git \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install whisper.cpp +WORKDIR /tmp +RUN git clone https://github.com/ggerganov/whisper.cpp.git && \ + cd whisper.cpp && \ + make && \ + cp main /usr/local/bin/whisper.cpp && \ + chmod +x /usr/local/bin/whisper.cpp + +# Download whisper model +RUN mkdir -p /models && \ + cd /models && \ + wget -q https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin && \ + mv ggml-base.en.bin ggml-base.bin + +# Set up application directory +WORKDIR /app + +# Copy requirements and install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY agent.py . + +# Create directories for videos +RUN mkdir -p /raw_movies /final_movies + +# Set environment variables with defaults +ENV RAW_DIR=/raw_movies \ + FINAL_DIR=/final_movies \ + WHISPER_MODEL=/models/ggml-base.bin \ + VAAPI_DEVICE=/dev/dri/renderD128 + +CMD ["python", "-u", "agent.py"] diff --git a/Dockerfile.init b/Dockerfile.init new file mode 100644 index 0000000..d85c732 --- /dev/null +++ b/Dockerfile.init @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +# Install curl, wget and other utilities +RUN apt-get update && apt-get install -y \ + curl \ + wget \ + jq \ + && rm -rf /var/lib/apt/lists/* + +# Install Python requests library +RUN pip install --no-cache-dir requests + +WORKDIR /app + +# The init script will be mounted +CMD ["/bin/bash"] diff --git a/PRODUCTION.md b/PRODUCTION.md new file mode 100644 index 0000000..0574bf3 --- /dev/null +++ b/PRODUCTION.md @@ -0,0 +1,875 @@ +# Production Deployment Guide + +Complete guide for running the Movie Scheduler in production environments. + +## Table of Contents +1. [Prerequisites](#prerequisites) +2. [Deployment Options](#deployment-options) +3. [Installation Methods](#installation-methods) +4. [Configuration](#configuration) +5. [Security](#security) +6. [Monitoring](#monitoring) +7. [Backup & Recovery](#backup--recovery) +8. [Maintenance](#maintenance) +9. [Troubleshooting](#troubleshooting) + +--- + +## Prerequisites + +### Hardware Requirements + +**Minimum:** +- CPU: 4 cores (for whisper.cpp and encoding) +- RAM: 4GB +- Storage: 100GB+ (depends on video library size) +- GPU: Intel/AMD with VAAPI support (optional but recommended) + +**Recommended:** +- CPU: 8+ cores +- RAM: 8GB+ +- Storage: 500GB+ SSD +- GPU: Modern Intel/AMD GPU with VAAPI + +### Software Requirements + +- **OS**: Linux (Ubuntu 20.04+, Debian 11+, RHEL 8+, or compatible) +- **Python**: 3.7+ +- **FFmpeg**: With VAAPI support +- **whisper.cpp**: Compiled and in PATH +- **Network**: Stable connection to NocoDB and RTMP server + +--- + +## Deployment Options + +### Option 1: Systemd Service (Recommended for bare metal) +✅ Direct hardware access (best VAAPI performance) +✅ Low overhead +✅ System integration +❌ Manual dependency management + +### Option 2: Docker Container (Recommended for most users) +✅ Isolated environment +✅ Easy updates +✅ Portable configuration +⚠️ Slight performance overhead +⚠️ Requires GPU passthrough for VAAPI + +### Option 3: Kubernetes/Orchestration +✅ High availability +✅ Auto-scaling +✅ Cloud-native +❌ Complex setup +❌ Overkill for single-instance deployment + +--- + +## Installation Methods + +### Method 1: Systemd Service Installation + +#### 1. Create Scheduler User + +```bash +# Create dedicated user for security +sudo useradd -r -s /bin/bash -d /opt/scheduler -m scheduler + +# Add to video group for GPU access +sudo usermod -aG video,render scheduler +``` + +#### 2. Install Dependencies + +```bash +# Install system packages +sudo apt-get update +sudo apt-get install -y python3 python3-pip python3-venv ffmpeg git build-essential + +# Install whisper.cpp +sudo -u scheduler git clone https://github.com/ggerganov/whisper.cpp.git /tmp/whisper.cpp +cd /tmp/whisper.cpp +make +sudo cp main /usr/local/bin/whisper.cpp +sudo chmod +x /usr/local/bin/whisper.cpp + +# Download whisper model +sudo mkdir -p /opt/models +cd /opt/models +sudo wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin +sudo mv ggml-base.en.bin ggml-base.bin +sudo chown -R scheduler:scheduler /opt/models +``` + +#### 3. Deploy Application + +```bash +# Create application directory +sudo mkdir -p /opt/scheduler +sudo chown scheduler:scheduler /opt/scheduler + +# Copy application files +sudo -u scheduler cp agent.py /opt/scheduler/ +sudo -u scheduler cp requirements.txt /opt/scheduler/ + +# Create Python virtual environment +sudo -u scheduler python3 -m venv /opt/scheduler/venv +sudo -u scheduler /opt/scheduler/venv/bin/pip install -r /opt/scheduler/requirements.txt +``` + +#### 4. Configure Storage + +```bash +# Create storage directories (adjust paths as needed) +sudo mkdir -p /mnt/storage/raw_movies +sudo mkdir -p /mnt/storage/final_movies +sudo chown -R scheduler:scheduler /mnt/storage +``` + +#### 5. Configure Service + +```bash +# Copy service file +sudo cp scheduler.service /etc/systemd/system/ + +# Create environment file with secrets +sudo mkdir -p /etc/scheduler +sudo nano /etc/scheduler/scheduler.env +``` + +Edit `/etc/scheduler/scheduler.env`: +```bash +NOCODB_URL=https://your-nocodb.com/api/v2/tables/YOUR_TABLE_ID/records +NOCODB_TOKEN=your_production_token +RTMP_SERVER=rtmp://your-rtmp-server.com/live/stream +RAW_DIR=/mnt/storage/raw_movies +FINAL_DIR=/mnt/storage/final_movies +WHISPER_MODEL=/opt/models/ggml-base.bin +``` + +Update `scheduler.service` to use the environment file: +```ini +# Replace Environment= lines with: +EnvironmentFile=/etc/scheduler/scheduler.env +``` + +#### 6. Enable and Start Service + +```bash +# Reload systemd +sudo systemctl daemon-reload + +# Enable service (start on boot) +sudo systemctl enable scheduler + +# Start service +sudo systemctl start scheduler + +# Check status +sudo systemctl status scheduler + +# View logs +sudo journalctl -u scheduler -f +``` + +--- + +### Method 2: Docker Deployment + +#### 1. Prepare Environment + +```bash +# Create project directory +mkdir -p /opt/scheduler +cd /opt/scheduler + +# Copy application files +cp agent.py requirements.txt Dockerfile docker-compose.prod.yml ./ + +# Create production environment file +cp .env.production.example .env.production +nano .env.production # Fill in your values +``` + +#### 2. Configure Storage + +```bash +# Ensure storage directories exist +mkdir -p /mnt/storage/raw_movies +mkdir -p /mnt/storage/final_movies + +# Download whisper model +mkdir -p /opt/models +cd /opt/models +wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin +mv ggml-base.en.bin ggml-base.bin +``` + +#### 3. Deploy Container + +```bash +cd /opt/scheduler + +# Build image +docker compose -f docker-compose.prod.yml build + +# Start service +docker compose -f docker-compose.prod.yml up -d + +# Check logs +docker compose -f docker-compose.prod.yml logs -f + +# Check status +docker compose -f docker-compose.prod.yml ps +``` + +#### 4. Enable Auto-Start + +```bash +# Create systemd service for docker compose +sudo nano /etc/systemd/system/scheduler-docker.service +``` + +```ini +[Unit] +Description=Movie Scheduler (Docker) +Requires=docker.service +After=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/opt/scheduler +ExecStart=/usr/bin/docker compose -f docker-compose.prod.yml up -d +ExecStop=/usr/bin/docker compose -f docker-compose.prod.yml down +TimeoutStartSec=0 + +[Install] +WantedBy=multi-user.target +``` + +```bash +sudo systemctl daemon-reload +sudo systemctl enable scheduler-docker +``` + +--- + +## Configuration + +### Essential Configuration + +#### NocoDB Connection + +```bash +# Get your table ID from NocoDB URL +# https://nocodb.com/nc/YOUR_BASE_ID/table_NAME +# API endpoint: https://nocodb.com/api/v2/tables/TABLE_ID/records + +# Generate API token in NocoDB: +# Account Settings → Tokens → Create Token +``` + +#### RTMP Server + +```bash +# For nginx-rtmp: +RTMP_SERVER=rtmp://your-server.com:1935/live/stream + +# For other RTMP servers, use their endpoint format +``` + +#### Storage Paths + +```bash +# Use separate fast storage for final videos (streaming) +RAW_DIR=/mnt/storage/raw_movies # Can be slower storage +FINAL_DIR=/mnt/fast-storage/final # Should be fast SSD + +# Ensure proper permissions +chown -R scheduler:scheduler /mnt/storage +chmod 755 /mnt/storage/raw_movies +chmod 755 /mnt/fast-storage/final +``` + +### Performance Tuning + +#### Sync Intervals + +```bash +# High-load scenario (many jobs, frequent updates) +NOCODB_SYNC_INTERVAL_SECONDS=120 # Check less often +WATCHDOG_CHECK_INTERVAL_SECONDS=15 # Check streams less often + +# Low-latency scenario (need fast response) +NOCODB_SYNC_INTERVAL_SECONDS=30 +WATCHDOG_CHECK_INTERVAL_SECONDS=5 + +# Default (balanced) +NOCODB_SYNC_INTERVAL_SECONDS=60 +WATCHDOG_CHECK_INTERVAL_SECONDS=10 +``` + +#### FFmpeg VAAPI + +```bash +# Find your VAAPI device +ls -la /dev/dri/ + +# Common devices: +# renderD128 - Primary GPU +# renderD129 - Secondary GPU + +# Test VAAPI +ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -i test.mp4 -f null - + +# Set in config +VAAPI_DEVICE=/dev/dri/renderD128 +``` + +--- + +## Security + +### Secrets Management + +**DO NOT hardcode secrets in files tracked by git!** + +#### Option 1: Environment Files (Simple) + +```bash +# Store secrets in protected file +sudo nano /etc/scheduler/scheduler.env +sudo chmod 600 /etc/scheduler/scheduler.env +sudo chown scheduler:scheduler /etc/scheduler/scheduler.env +``` + +#### Option 2: Secrets Management Tools + +```bash +# Using Vault +export NOCODB_TOKEN=$(vault kv get -field=token secret/scheduler/nocodb) + +# Using AWS Secrets Manager +export NOCODB_TOKEN=$(aws secretsmanager get-secret-value --secret-id scheduler/nocodb --query SecretString --output text) + +# Using Docker Secrets (Swarm/Kubernetes) +# Mount secrets as files and read in application +``` + +### Filesystem Permissions + +```bash +# Application directory +chown -R scheduler:scheduler /opt/scheduler +chmod 750 /opt/scheduler + +# Storage directories +chown -R scheduler:scheduler /mnt/storage +chmod 755 /mnt/storage/raw_movies # Read-only for scheduler +chmod 755 /mnt/storage/final_movies # Read-write for scheduler + +# Database file +chmod 600 /opt/scheduler/scheduler.db +chown scheduler:scheduler /opt/scheduler/scheduler.db +``` + +### Network Security + +```bash +# Firewall rules (if scheduler runs on separate server) +# Only allow outbound connections to NocoDB and RTMP + +sudo ufw allow out to YOUR_NOCODB_IP port 443 # HTTPS +sudo ufw allow out to YOUR_RTMP_IP port 1935 # RTMP +sudo ufw default deny outgoing # Deny all other outbound (optional) +``` + +### Regular Updates + +```bash +# Update system packages weekly +sudo apt-get update && sudo apt-get upgrade + +# Update Python dependencies +sudo -u scheduler /opt/scheduler/venv/bin/pip install --upgrade requests + +# Rebuild whisper.cpp quarterly (for performance improvements) +``` + +--- + +## Monitoring + +### Service Health + +#### Systemd Monitoring + +```bash +# Check service status +systemctl status scheduler + +# View recent logs +journalctl -u scheduler -n 100 + +# Follow logs in real-time +journalctl -u scheduler -f + +# Check for errors in last hour +journalctl -u scheduler --since "1 hour ago" | grep ERROR + +# Service restart count +systemctl show scheduler | grep NRestarts +``` + +#### Docker Monitoring + +```bash +# Container status +docker compose -f docker-compose.prod.yml ps + +# Resource usage +docker stats movie_scheduler + +# Logs +docker compose -f docker-compose.prod.yml logs --tail=100 -f + +# Health check status +docker inspect movie_scheduler | jq '.[0].State.Health' +``` + +### Database Monitoring + +```bash +# Check job status +sqlite3 /opt/scheduler/scheduler.db "SELECT prep_status, play_status, COUNT(*) FROM jobs GROUP BY prep_status, play_status;" + +# Active streams +sqlite3 /opt/scheduler/scheduler.db "SELECT nocodb_id, title, play_status, stream_retry_count FROM jobs WHERE play_status='streaming';" + +# Failed jobs +sqlite3 /opt/scheduler/scheduler.db "SELECT nocodb_id, title, prep_status, play_status, log FROM jobs WHERE prep_status='failed' OR play_status='failed';" + +# Database size +ls -lh /opt/scheduler/scheduler.db +``` + +### Automated Monitoring Script + +Create `/opt/scheduler/monitor.sh`: + +```bash +#!/bin/bash + +LOG_FILE="/var/log/scheduler-monitor.log" +DB_PATH="/opt/scheduler/scheduler.db" + +echo "=== Scheduler Monitor - $(date) ===" >> "$LOG_FILE" + +# Check if service is running +if systemctl is-active --quiet scheduler; then + echo "✓ Service is running" >> "$LOG_FILE" +else + echo "✗ Service is DOWN" >> "$LOG_FILE" + # Send alert (email, Slack, etc.) + systemctl start scheduler +fi + +# Check for failed jobs +FAILED=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE prep_status='failed' OR play_status='failed';") +if [ "$FAILED" -gt 0 ]; then + echo "⚠ Found $FAILED failed jobs" >> "$LOG_FILE" + # Send alert +fi + +# Check active streams +STREAMING=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE play_status='streaming';") +echo "Active streams: $STREAMING" >> "$LOG_FILE" + +# Check disk space +DISK_USAGE=$(df -h /mnt/storage | tail -1 | awk '{print $5}' | sed 's/%//') +if [ "$DISK_USAGE" -gt 90 ]; then + echo "⚠ Disk usage is ${DISK_USAGE}%" >> "$LOG_FILE" + # Send alert +fi + +echo "" >> "$LOG_FILE" +``` + +```bash +# Make executable +chmod +x /opt/scheduler/monitor.sh + +# Add to crontab (check every 5 minutes) +(crontab -l 2>/dev/null; echo "*/5 * * * * /opt/scheduler/monitor.sh") | crontab - +``` + +### External Monitoring + +#### Prometheus + Grafana + +Export metrics using node_exporter or custom exporter: + +```bash +# Install node_exporter for system metrics +# Create custom exporter for job metrics from database +# Set up Grafana dashboard +``` + +#### Uptime Monitoring + +Use services like: +- UptimeRobot +- Pingdom +- Datadog + +Monitor: +- Service availability +- RTMP server connectivity +- NocoDB API accessibility + +--- + +## Backup & Recovery + +### What to Backup + +1. **Database** (scheduler.db) - Critical +2. **Configuration** (.env.production or /etc/scheduler/scheduler.env) - Critical +3. **Final videos** (if you want to keep processed videos) +4. **Logs** (optional, for forensics) + +### Backup Script + +Create `/opt/scheduler/backup.sh`: + +```bash +#!/bin/bash + +BACKUP_DIR="/backup/scheduler" +DATE=$(date +%Y%m%d_%H%M%S) +DB_PATH="/opt/scheduler/scheduler.db" +CONFIG_PATH="/etc/scheduler/scheduler.env" + +# Create backup directory +mkdir -p "$BACKUP_DIR" + +# Backup database +cp "$DB_PATH" "$BACKUP_DIR/scheduler_${DATE}.db" + +# Backup config (careful with secrets!) +cp "$CONFIG_PATH" "$BACKUP_DIR/config_${DATE}.env" + +# Compress old backups +find "$BACKUP_DIR" -name "*.db" -mtime +7 -exec gzip {} \; + +# Delete backups older than 30 days +find "$BACKUP_DIR" -name "*.db.gz" -mtime +30 -delete +find "$BACKUP_DIR" -name "*.env" -mtime +30 -delete + +# Optional: Upload to S3/cloud storage +# aws s3 sync "$BACKUP_DIR" s3://your-bucket/scheduler-backups/ + +echo "Backup completed: $BACKUP_DIR/scheduler_${DATE}.db" +``` + +```bash +# Make executable +chmod +x /opt/scheduler/backup.sh + +# Run daily at 2 AM +(crontab -l 2>/dev/null; echo "0 2 * * * /opt/scheduler/backup.sh") | crontab - +``` + +### Recovery Procedure + +#### 1. Restore from Backup + +```bash +# Stop service +sudo systemctl stop scheduler + +# Restore database +cp /backup/scheduler/scheduler_YYYYMMDD_HHMMSS.db /opt/scheduler/scheduler.db +chown scheduler:scheduler /opt/scheduler/scheduler.db + +# Restore config if needed +cp /backup/scheduler/config_YYYYMMDD_HHMMSS.env /etc/scheduler/scheduler.env + +# Start service +sudo systemctl start scheduler +``` + +#### 2. Disaster Recovery (Full Rebuild) + +If server is completely lost: + +1. Provision new server +2. Follow installation steps above +3. Restore database and config from backup +4. Restart service +5. Verify jobs are picked up + +**Recovery Time Objective (RTO):** 30-60 minutes +**Recovery Point Objective (RPO):** Up to 24 hours (with daily backups) + +--- + +## Maintenance + +### Routine Tasks + +#### Daily +- ✓ Check service status +- ✓ Review error logs +- ✓ Check failed jobs + +#### Weekly +- ✓ Review disk space +- ✓ Check database size +- ✓ Clean up old processed videos (if not needed) + +#### Monthly +- ✓ Update system packages +- ✓ Review and optimize database +- ✓ Test backup restoration +- ✓ Review and rotate logs + +### Database Maintenance + +```bash +# Vacuum database (reclaim space, optimize) +sqlite3 /opt/scheduler/scheduler.db "VACUUM;" + +# Analyze database (update statistics) +sqlite3 /opt/scheduler/scheduler.db "ANALYZE;" + +# Clean up old completed jobs (optional) +sqlite3 /opt/scheduler/scheduler.db "DELETE FROM jobs WHERE play_status='done' AND datetime(run_at) < datetime('now', '-30 days');" +``` + +### Log Rotation + +For systemd (automatic via journald): +```bash +# Configure in /etc/systemd/journald.conf +SystemMaxUse=1G +RuntimeMaxUse=100M +``` + +For Docker: +```yaml +# Already configured in docker-compose.prod.yml +logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "5" +``` + +### Video Cleanup + +```bash +# Clean up old final videos (adjust retention as needed) +find /mnt/storage/final_movies -name "*.mp4" -mtime +7 -delete + +# Or move to archive +find /mnt/storage/final_movies -name "*.mp4" -mtime +7 -exec mv {} /mnt/archive/ \; +``` + +--- + +## Troubleshooting + +### Service Won't Start + +```bash +# Check service status +systemctl status scheduler + +# Check logs for errors +journalctl -u scheduler -n 50 + +# Common issues: +# 1. Missing environment variables +grep -i "error" /var/log/syslog | grep scheduler + +# 2. Permission issues +ls -la /opt/scheduler +ls -la /mnt/storage + +# 3. GPU access issues +ls -la /dev/dri/ +groups scheduler # Should include 'video' and 'render' +``` + +### Streams Keep Failing + +```bash +# Test RTMP server manually +ffmpeg -re -i test.mp4 -c copy -f flv rtmp://your-server/live/stream + +# Check network connectivity +ping your-rtmp-server.com +telnet your-rtmp-server.com 1935 + +# Review stream logs +journalctl -u scheduler | grep -A 10 "Stream crashed" + +# Check retry count +sqlite3 /opt/scheduler/scheduler.db "SELECT nocodb_id, title, stream_retry_count FROM jobs WHERE play_status='streaming';" +``` + +### High CPU/Memory Usage + +```bash +# Check resource usage +top -u scheduler + +# Or for Docker +docker stats movie_scheduler + +# Common causes: +# 1. Large video file encoding - normal, wait for completion +# 2. whisper.cpp using all cores - normal +# 3. Multiple prep jobs running - adjust or wait + +# Limit resources if needed (systemd) +systemctl edit scheduler +# Add: +[Service] +CPUQuota=200% +MemoryMax=4G +``` + +### Database Locked Errors + +```bash +# Check for stale locks +lsof /opt/scheduler/scheduler.db + +# Kill stale processes if needed +# Restart service +systemctl restart scheduler +``` + +### VAAPI Not Working + +```bash +# Verify VAAPI support +vainfo + +# Test FFmpeg VAAPI +ffmpeg -hwaccels + +# Check permissions +ls -la /dev/dri/renderD128 +groups scheduler # Should include 'video' or 'render' + +# Fallback to software encoding +# Comment out VAAPI_DEVICE in config +# Encoding will use CPU (slower but works) +``` + +--- + +## Performance Optimization + +### Hardware Acceleration + +```bash +# Verify GPU usage during encoding +intel_gpu_top # For Intel GPUs +radeontop # For AMD GPUs + +# If GPU not being used, check: +# 1. VAAPI device path correct +# 2. User has GPU permissions +# 3. FFmpeg compiled with VAAPI support +``` + +### Storage Performance + +```bash +# Use SSD for final videos (they're streamed frequently) +# Use HDD for raw videos (accessed once for processing) + +# Test disk performance +dd if=/dev/zero of=/mnt/storage/test bs=1M count=1024 oflag=direct +rm /mnt/storage/test +``` + +### Network Optimization + +```bash +# For better streaming reliability +# 1. Use dedicated network for RTMP +# 2. Enable QoS for streaming traffic +# 3. Consider local RTMP relay + +# Test network throughput +iperf3 -c your-rtmp-server.com +``` + +--- + +## Production Checklist + +Before going live: + +- [ ] Secrets stored securely (not in git) +- [ ] Service auto-starts on boot +- [ ] Backups configured and tested +- [ ] Monitoring configured +- [ ] Logs being rotated +- [ ] Disk space alerts configured +- [ ] Test recovery procedure +- [ ] Document runbook for on-call +- [ ] GPU permissions verified +- [ ] RTMP connectivity tested +- [ ] NocoDB API tested +- [ ] Process one test video end-to-end +- [ ] Verify streaming watchdog works +- [ ] Test service restart during streaming +- [ ] Configure alerting for failures + +--- + +## Support & Updates + +### Getting Updates + +```bash +# Git-based deployment +cd /opt/scheduler +git pull origin main +systemctl restart scheduler + +# Docker-based deployment +cd /opt/scheduler +docker compose -f docker-compose.prod.yml pull +docker compose -f docker-compose.prod.yml up -d +``` + +### Reporting Issues + +Include in bug reports: +- Service logs: `journalctl -u scheduler -n 100` +- Database state: `sqlite3 scheduler.db ".dump jobs"` +- System info: `uname -a`, `python3 --version`, `ffmpeg -version` +- Configuration (redact secrets!) + +--- + +## Additional Resources + +- FFmpeg VAAPI Guide: https://trac.ffmpeg.org/wiki/Hardware/VAAPI +- whisper.cpp: https://github.com/ggerganov/whisper.cpp +- NocoDB API: https://docs.nocodb.com +- Systemd Documentation: https://www.freedesktop.org/software/systemd/man/ + +--- + +## License + +This production guide is provided as-is. Test thoroughly in staging before production deployment. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4dc6702 --- /dev/null +++ b/README.md @@ -0,0 +1,314 @@ +# Movie Scheduler + +Automated movie scheduler that reads from NocoDB, generates subtitles with whisper.cpp, burns them into videos with VAAPI encoding, and streams at scheduled times. + +## Quick Links + +- **[Production Deployment Guide](PRODUCTION.md)** - Complete guide for production setup +- **[Testing Guide](TESTING.md)** - Docker-based test environment +- **[Development Setup](#installation)** - Local development setup + +## Features + +- **NocoDB Integration**: Syncs job schedules from NocoDB database +- **Automatic Subtitle Generation**: Uses whisper.cpp to generate SRT subtitles +- **Hardware-Accelerated Encoding**: Burns subtitles into videos using VAAPI (h264_vaapi) +- **Scheduled Streaming**: Streams videos to RTMP server at scheduled times +- **Robust Error Handling**: Automatic retries with exponential backoff +- **Comprehensive Logging**: Tracks all operations in database log column +- **Process Management**: Properly tracks and manages streaming processes + +## Requirements + +- Python 3.7+ +- `requests` library +- `whisper.cpp` installed and available in PATH +- `ffmpeg` with VAAPI support +- VAAPI-capable hardware (AMD/Intel GPU) +- NocoDB instance with scheduled jobs table + +## Installation + +1. Clone or download this repository + +2. Install Python dependencies: + ```bash + pip install requests + ``` + +3. Ensure whisper.cpp is installed: + ```bash + # Follow whisper.cpp installation instructions + # Make sure the binary is in your PATH + which whisper.cpp + ``` + +4. Verify FFmpeg has VAAPI support: + ```bash + ffmpeg -hwaccels | grep vaapi + ``` + +## Configuration + +1. Copy the example environment file: + ```bash + cp .env.example .env + ``` + +2. Edit `.env` and set the required variables: + ```bash + # Required + export NOCODB_URL="https://nocodb/api/v2/tables/XXXX/records" + export NOCODB_TOKEN="your_token_here" + export RTMP_SERVER="rtmp://your_server/live" + + # Optional (defaults provided) + export RAW_DIR="/root/surowe_filmy" + export FINAL_DIR="/root/przygotowane_filmy" + export WHISPER_MODEL="/root/models/ggml-base.bin" + export VAAPI_DEVICE="/dev/dri/renderD128" + export STREAM_GRACE_PERIOD_MINUTES="15" + export NOCODB_SYNC_INTERVAL_SECONDS="60" + export WATCHDOG_CHECK_INTERVAL_SECONDS="10" + ``` + +3. Load the environment variables: + ```bash + source .env + ``` + +## NocoDB Table Structure + +Your NocoDB table should have the following fields: + +- `Id` - Unique identifier (Text/Primary Key) +- `Title` - Movie title matching filename (Text) +- `Date` - Scheduled run time (DateTime in ISO format) + +The scheduler will automatically create additional columns in the local SQLite database: +- `prep_at` - Preparation time (6 hours before run_at) +- `prep_status` - Preparation status (pending/done/failed) +- `play_status` - Streaming status (pending/done/failed) +- `raw_path` - Path to source video file +- `final_path` - Path to converted video with subtitles +- `log` - Detailed operation log + +## Usage + +### Running the Scheduler + +```bash +# Ensure environment variables are set +source .env + +# Run the scheduler +python agent.py +``` + +The scheduler will: +1. Sync jobs from NocoDB every 60 seconds (configurable) +2. Check stream health every 10 seconds (configurable) +3. Prepare videos 6 hours before their scheduled time: + - Find matching video file in RAW_DIR + - Generate subtitles with whisper.cpp + - Encode video with burned subtitles using VAAPI +4. Stream videos at their scheduled time to RTMP server + +### Stopping the Scheduler + +Press `Ctrl+C` to gracefully stop the scheduler. It will: +- Stop any active streaming process +- Close the database connection +- Exit cleanly + +### Restart & Recovery Behavior + +The scheduler handles restarts, power outages, and downtime gracefully: + +**On Startup:** +- ✅ Checks for overdue prep jobs and processes them immediately +- ⚠️ Skips jobs where the streaming time has already passed +- 📊 Logs recovery status (overdue/skipped jobs found) + +**Grace Period for Late Streaming:** +- ⏰ If prep is done and streaming time is overdue by **up to 15 minutes**, will still start streaming +- ⚠️ Jobs more than 15 minutes late are marked as 'skipped' +- 📝 Late starts are logged with exact delay time + +**Recovery Scenarios:** + +1. **Short Outage (prep time missed, but streaming time not reached)** + - Movie scheduled for 21:00 (prep at 15:00) + - System down from 14:00-16:00 + - ✅ Restarts at 16:00: immediately preps the movie + - ✅ Streams normally at 21:00 + +2. **Late Start Within Grace Period (< 15 minutes)** + - Movie scheduled for 21:00 (prep completed at 15:00) + - System down from 20:00-21:10 + - ✅ Restarts at 21:10: starts streaming immediately (10 minutes late) + - 📝 Logged: "Starting stream 10.0 minutes late" + +3. **Too Late to Stream (> 15 minutes)** + - Movie scheduled for 21:00 (prep completed) + - System down from 20:00-21:20 + - ⚠️ Restarts at 21:20: marks streaming as 'skipped' (>15 min late) + - 📝 Logged: "Streaming skipped - more than 15 minutes late" + +4. **Long Outage (both prep and streaming times passed, >15 min)** + - Movie scheduled for 21:00 (prep at 15:00) + - System down from 14:00-22:00 + - ⚠️ Restarts at 22:00: marks entire job as 'skipped' + - 📝 Logged: "Job skipped - streaming time already passed" + +5. **Crash During Processing** + - System crashes during subtitle generation or encoding + - ✅ On restart: retries from the beginning with full retry logic + - All operations are idempotent (safe to re-run) + +**Database Persistence:** +- Job status stored in `scheduler.db` survives restarts +- Completed preps are never re-done +- Failed jobs stay marked as 'failed' (can be reset manually to 'pending' if needed) + +## File Naming + +Place your raw video files in `RAW_DIR` with names that contain the title from NocoDB. + +Example: +- NocoDB Title: `"The Matrix"` +- Valid filenames: `The Matrix.mkv`, `the-matrix-1999.mp4`, `[1080p] The Matrix.avi` + +The scheduler uses glob matching to find files containing the title. + +## Streaming Watchdog + +The scheduler includes an active watchdog that monitors streaming: + +**Stream Monitoring (checked every 10 seconds):** +- ✅ Detects if ffmpeg stream crashes or exits unexpectedly +- ✅ Automatically restarts stream at the correct playback position +- ✅ Calculates elapsed time since stream start +- ✅ Seeks to correct position using `ffmpeg -ss` flag +- ✅ Limits restarts to 10 attempts to prevent infinite loops +- ✅ Marks stream as 'done' when video completes normally +- ✅ Marks stream as 'failed' after 10 failed restart attempts + +**How It Works:** +1. Stream starts and status set to 'streaming' (not 'done') +2. Watchdog checks every 10 seconds if stream process is running +3. If crashed: calculates elapsed time and restarts with seek +4. If completed normally (exit code 0): marks as 'done' +5. If video duration exceeded: marks as 'done' + +**Example:** +``` +Stream starts at 21:00:00 +Stream crashes at 21:05:30 (5.5 minutes in) +Watchdog detects crash within 10 seconds +Restarts stream with: ffmpeg -ss 330 -i video.mp4 +Stream resumes from 5:30 mark +``` + +## Error Handling + +The scheduler includes robust error handling: + +- **Automatic Retries**: Failed operations are retried up to 3 times with exponential backoff +- **Database Logging**: All operations and errors are logged to the `log` column +- **Status Tracking**: Jobs are marked as 'failed' if all retries are exhausted +- **File Verification**: Checks that subtitle files and encoded videos were created successfully + +## Monitoring + +### Check Job Status + +Query the SQLite database: +```bash +sqlite3 scheduler.db "SELECT nocodb_id, title, prep_status, play_status FROM jobs;" +``` + +### View Job Logs + +```bash +sqlite3 scheduler.db "SELECT nocodb_id, title, log FROM jobs WHERE nocodb_id='YOUR_JOB_ID';" +``` + +### Console Output + +The scheduler logs to stdout with timestamps: +- INFO: Normal operations +- WARNING: Retry attempts +- ERROR: Failures + +## Troubleshooting + +### Configuration Validation Failed +- Ensure all required environment variables are set +- Check that NOCODB_URL is accessible +- Verify NOCODB_TOKEN has proper permissions + +### No Files Found Matching Title +- Check that video files exist in RAW_DIR +- Ensure filenames contain the NocoDB title (case-insensitive glob matching) +- Review the log column in the database for exact error messages + +### Subtitle Generation Failed +- Verify whisper.cpp is installed: `which whisper.cpp` +- Check WHISPER_MODEL path is correct +- Ensure the model file exists and is readable + +### Video Encoding Failed +- Confirm VAAPI is available: `ffmpeg -hwaccels | grep vaapi` +- Check VAAPI device exists: `ls -l /dev/dri/renderD128` +- Verify subtitle file was created successfully +- Review FFmpeg error output in the log column + +### Streaming Failed +- Test RTMP server connectivity: `ffmpeg -re -i test.mp4 -c copy -f flv rtmp://your_server/live` +- Ensure final video file exists +- Check network connectivity to RTMP server + +## Development + +### Database Schema + +```sql +CREATE TABLE jobs ( + nocodb_id TEXT PRIMARY KEY, + title TEXT, + run_at TIMESTAMP, + prep_at TIMESTAMP, + raw_path TEXT, + final_path TEXT, + prep_status TEXT, -- pending/done/failed/skipped + play_status TEXT, -- pending/streaming/done/failed/skipped + log TEXT, + stream_start_time TIMESTAMP, -- When streaming started (for restart seek) + stream_retry_count INTEGER -- Number of stream restart attempts +); +``` + +**Status Values:** +- `play_status='streaming'`: Stream is actively running +- `play_status='done'`: Stream completed successfully +- `play_status='failed'`: Stream failed after 10 restart attempts +- `play_status='skipped'`: Too late to stream (>15 min past scheduled time) + +### Customization + +Use environment variables to customize: +- `NOCODB_SYNC_INTERVAL_SECONDS` - How often to check NocoDB for new jobs (default: 60) +- `WATCHDOG_CHECK_INTERVAL_SECONDS` - How often to monitor streams (default: 10) +- `STREAM_GRACE_PERIOD_MINUTES` - Late start tolerance (default: 15) + +Edit `agent.py` to customize: +- Retry attempts and delay (agent.py:99, 203, 232) +- Preparation time offset (currently 6 hours, see sync function) +- Stream restart limit (agent.py:572) +- Subtitle styling (agent.py:246) +- Video encoding parameters (agent.py:248-250) + +## License + +This project is provided as-is without warranty. diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..8644f36 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,486 @@ +# Movie Scheduler - Testing Environment + +Complete Docker-based testing environment for the movie scheduler. Fire and forget - just run one command and everything is set up automatically! + +## Quick Start + +```bash +./start-test-environment.sh +``` + +That's it! The script will: +1. Build all Docker containers +2. Start PostgreSQL, NocoDB, and RTMP server +3. Initialize NocoDB with test data +4. Download a test video (Big Buck Bunny) +5. Start the scheduler + +## Services + +After running the startup script, you'll have: + +### NocoDB Dashboard +- **URL**: http://localhost:8080 +- **Email**: admin@test.com +- **Password**: admin123 + +Login and view the "Movies" table to see scheduled movies and their status. + +### RTMP Streaming Server +- **Stream URL**: rtmp://localhost:1935/live/stream +- **Statistics**: http://localhost:8081/stat + +This is where processed movies will be streamed. + +### Movie Scheduler +Runs in the background, automatically: +- Syncs movies from NocoDB every 60 seconds (configurable) +- Monitors stream health every 10 seconds (configurable) +- Prepares videos 6 hours before scheduled time +- Streams at the scheduled time + +## Testing the Workflow + +### 1. View Initial Data + +The test environment comes with a pre-loaded movie: +- **Title**: BigBuckBunny +- **Scheduled**: 5 minutes from startup + +Open NocoDB to see this entry. + +### 2. Monitor the Scheduler + +Watch the scheduler logs: +```bash +docker compose logs -f scheduler +``` + +You'll see: +- Syncing from NocoDB +- Preparation jobs (if within 6 hours of scheduled time) +- Subtitle generation +- Video encoding +- Streaming start + +### 3. Check the Database + +Query the local SQLite database: +```bash +sqlite3 scheduler.db "SELECT nocodb_id, title, prep_status, play_status FROM jobs;" +``` + +View detailed logs: +```bash +sqlite3 scheduler.db "SELECT title, log FROM jobs WHERE title='BigBuckBunny';" +``` + +### 4. Watch the Stream + +When a movie starts streaming, watch it with VLC: +```bash +vlc rtmp://localhost:1935/live/stream +``` + +Or use ffplay: +```bash +ffplay rtmp://localhost:1935/live/stream +``` + +### 5. Add Your Own Movies + +#### Option A: Through NocoDB UI + +1. Open http://localhost:8080 +2. Navigate to the Movies table +3. Click "+ Add New Row" +4. Fill in: + - **Title**: Name matching your video file + - **Date**: Schedule time (ISO format: 2024-01-21T20:00:00) + - Other fields are optional +5. Place video file in `raw_movies` volume + +#### Option B: Add Video File Manually + +Copy a video to the raw movies volume: +```bash +# Find the volume name +docker volume ls | grep raw_movies + +# Copy your video +docker run --rm -v scheduler_raw_movies:/data -v $(pwd):/host alpine \ + cp /host/your-movie.mp4 /data/ +``` + +Then add the entry in NocoDB with matching Title. + +## Testing Restart & Recovery + +### Test Overdue Job Processing + +1. Add a movie scheduled for 15 minutes from now +2. Stop the scheduler: + ```bash + docker compose stop scheduler + ``` +3. Wait 20 minutes (past the prep time) +4. Restart the scheduler: + ```bash + docker compose start scheduler + docker compose logs -f scheduler + ``` +5. Watch it immediately process the overdue prep job + +### Test 15-Minute Grace Period for Streaming + +1. Add a movie scheduled for 5 minutes from now (with prep already done) +2. Stop the scheduler after prep completes: + ```bash + docker compose stop scheduler + ``` +3. Wait 10 minutes (past streaming time but within 15-min grace period) +4. Restart and watch logs: + ```bash + docker compose start scheduler + docker compose logs -f scheduler + ``` +5. Stream should start immediately with "Starting stream X.X minutes late" message + +### Test Skipping Late Streams (>15 minutes) + +1. Add a movie scheduled for 5 minutes from now +2. Let prep complete, then stop the scheduler: + ```bash + docker compose stop scheduler + ``` +3. Wait 20 minutes (past the 15-minute grace period) +4. Restart and watch logs: + ```bash + docker compose start scheduler + docker compose logs -f scheduler + ``` +5. Job should be marked as 'skipped' with "more than 15 minutes late" + +### Test Skipping Unprepared Expired Jobs + +1. Add a movie scheduled for 5 minutes from now +2. Stop the scheduler before prep starts: + ```bash + docker compose stop scheduler + ``` +3. Wait 10 minutes (past both prep and streaming time) +4. Restart and watch logs: + ```bash + docker compose start scheduler + docker compose logs -f scheduler + ``` +5. Job should be marked as 'skipped' (too late to prep) + +### Test Recovery from Crash During Processing + +1. Start processing a large video file +2. Kill the scheduler during encoding: + ```bash + docker compose kill scheduler + ``` +3. Restart: + ```bash + docker compose start scheduler + ``` +4. Watch it retry the entire operation from the beginning + +## Testing Stream Watchdog + +### Test Stream Crash and Auto-Restart + +1. Start a stream: + ```bash + docker compose logs -f scheduler + ``` + +2. Find the streaming ffmpeg process and kill it: + ```bash + # In another terminal + docker compose exec scheduler ps aux | grep ffmpeg + docker compose exec scheduler kill -9 + ``` + +3. Watch the scheduler logs - within 10 seconds it should: + - Detect the crash + - Calculate elapsed playback time + - Restart with seek to correct position + - Log: "Restarting stream at position X.Xs" + +### Test Stream Restart with Correct Position + +1. Start a test stream with a longer video +2. Let it run for 2-3 minutes +3. Kill the ffmpeg process +4. Watch logs confirm restart with seek: + ``` + Restarting stream for job XXX at position 180.0s (attempt 2) + ``` +5. Use VLC to confirm playback resumed at correct position + +### Test Stream Completion Detection + +1. Use a short test video (1-2 minutes) +2. Watch stream until completion +3. Check logs - should show: + ``` + Stream completed successfully for job XXX + ``` +4. Check database - play_status should be 'done' + +### Test Retry Limit + +1. Break the RTMP server or use invalid RTMP URL +2. Start a stream - it will crash immediately +3. Watch the scheduler attempt 10 restarts +4. After 10 attempts, should mark as 'failed': + ``` + Stream retry limit exceeded for job XXX + ERROR: Stream failed after 10 restart attempts + ``` + +### Test Network Interruption Recovery + +1. Start a stream +2. Stop the RTMP server: + ```bash + docker compose stop rtmp + ``` +3. Watch stream fail and retry +4. Restart RTMP: + ```bash + docker compose start rtmp + ``` +5. Stream should resume at correct position + +## Testing Error Handling + +### Test Retry Logic + +1. Stop the RTMP server: + ```bash + docker compose stop rtmp + ``` + +2. Watch the scheduler handle the failure and retry: + ```bash + docker compose logs -f scheduler + ``` + +3. Restart RTMP: + ```bash + docker compose start rtmp + ``` + +### Test Missing File + +1. Add a movie entry in NocoDB with a title that doesn't match any file +2. Set the Date to trigger immediately +3. Watch the scheduler log the error and mark it as failed + +### Test Subtitle Generation Failure + +1. Add an invalid video file (corrupted or wrong format) +2. Watch the scheduler attempt retries with exponential backoff + +## Viewing Results + +### Processed Videos + +List final videos with burned-in subtitles: +```bash +docker run --rm -v scheduler_final_movies:/data alpine ls -lh /data +``` + +### Raw Videos + +List source videos: +```bash +docker run --rm -v scheduler_raw_movies:/data alpine ls -lh /data +``` + +### RTMP Recordings (if enabled) + +Recordings are saved to the rtmp_recordings volume: +```bash +docker run --rm -v scheduler_rtmp_recordings:/data alpine ls -lh /data/recordings +``` + +## Customization + +### Change Test Video Scheduling + +Edit `init-data.sh` before starting, around line 130: +```python +"Date": (datetime.now() + timedelta(minutes=5)).isoformat(), +``` + +Change `minutes=5` to adjust when the test movie is scheduled. + +### Use Different NocoDB Credentials + +Edit `docker-compose.yml`: +```yaml +environment: + NOCODB_EMAIL: "your@email.com" + NOCODB_PASSWORD: "yourpassword" +``` + +### Test with Real NocoDB Instance + +Comment out the `nocodb` and `postgres` services in `docker-compose.yml` and update the `init` service environment: +```yaml +environment: + NOCODB_URL: "https://your-nocodb-instance.com" + NOCODB_EMAIL: "your@email.com" + NOCODB_PASSWORD: "yourpassword" +``` + +## Troubleshooting + +### Services Won't Start + +Check if ports are already in use: +```bash +lsof -i :8080 # NocoDB +lsof -i :1935 # RTMP +lsof -i :8081 # RTMP stats +``` + +### Initialization Fails + +View init logs: +```bash +docker compose logs init +``` + +Common issues: +- NocoDB not ready yet (increase wait time in init-data.sh) +- Network connectivity issues +- Insufficient disk space + +### Scheduler Not Processing + +Check if it's running: +```bash +docker compose ps scheduler +``` + +View logs for errors: +```bash +docker compose logs scheduler +``` + +Restart if needed: +```bash +docker compose restart scheduler +``` + +### Video Download Fails + +The init script downloads Big Buck Bunny from Google's test video bucket. If this fails: +1. Check internet connectivity +2. Manually download and copy to raw_movies volume +3. Or use your own test video + +### VAAPI Not Available + +The Docker container runs without GPU access by default (CPU encoding fallback). To enable VAAPI: + +Edit `docker-compose.yml` scheduler service: +```yaml +scheduler: + devices: + - /dev/dri:/dev/dri + group_add: + - video +``` + +## Cleanup + +### Stop Everything +```bash +docker compose down +``` + +### Remove All Data (including volumes) +```bash +docker compose down -v +``` + +This removes: +- All containers +- All volumes (database, videos, recordings) +- Network + +### Keep Database, Remove Containers +```bash +docker compose down +# Volumes persist - next startup will reuse existing data +``` + +## Architecture + +``` +┌─────────────┐ +│ PostgreSQL │ +└──────┬──────┘ + │ +┌──────▼──────┐ ┌─────────────┐ +│ NocoDB │◄─────┤ Init │ Downloads test video +└──────┬──────┘ │ Container │ Creates test data + │ └─────────────┘ + │ +┌──────▼──────┐ +│ Scheduler │ Reads from NocoDB +└──────┬──────┘ Generates subtitles + │ Encodes videos + │ Manages streaming + │ +┌──────▼──────┐ +│ RTMP Server │ Streams processed videos +└─────────────┘ +``` + +## Production Considerations + +This test environment uses: +- Embedded PostgreSQL (fine for testing) +- Local volumes (fine for testing) +- Simple authentication (change for production) +- No SSL/TLS (add for production) +- CPU-only encoding (enable VAAPI for production) + +For production deployment: +1. Use external PostgreSQL database +2. Configure proper authentication and secrets +3. Enable SSL/TLS for NocoDB +4. Mount persistent storage for videos +5. Enable GPU acceleration (VAAPI) +6. Set up proper monitoring and logging +7. Configure backup for NocoDB database + +## Next Steps + +Once you've tested the environment: +1. Review the logs to understand the workflow +2. Try adding your own movies +3. Test error scenarios +4. Adjust timing and configuration +5. Deploy to production with proper setup + +## Support + +For issues with: +- **Scheduler code**: Check `agent.py` and logs +- **Docker setup**: Check `docker-compose.yml` and service logs +- **NocoDB**: Visit https://docs.nocodb.com +- **RTMP streaming**: Check nginx-rtmp documentation + +## License + +This test environment is provided as-is for development and testing purposes. diff --git a/agent.py b/agent.py new file mode 100644 index 0000000..51bea1c --- /dev/null +++ b/agent.py @@ -0,0 +1,684 @@ +import requests +import sqlite3 +import time +import subprocess +import hashlib +import os +import sys +import logging +from datetime import datetime, timedelta +from pathlib import Path +from functools import wraps + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +# Load configuration from environment variables +NOCODB_URL = os.getenv("NOCODB_URL") +NOCODB_TOKEN = os.getenv("NOCODB_TOKEN") +RTMP_SERVER = os.getenv("RTMP_SERVER") +RAW_DIR = Path(os.getenv("RAW_DIR", "/root/surowe_filmy")) +FINAL_DIR = Path(os.getenv("FINAL_DIR", "/root/przygotowane_filmy")) +WHISPER_MODEL = os.getenv("WHISPER_MODEL", "/root/models/ggml-base.bin") +VAAPI_DEVICE = os.getenv("VAAPI_DEVICE", "/dev/dri/renderD128") +STREAM_GRACE_PERIOD_MINUTES = int(os.getenv("STREAM_GRACE_PERIOD_MINUTES", "15")) + +# Timing configuration +NOCODB_SYNC_INTERVAL_SECONDS = int(os.getenv("NOCODB_SYNC_INTERVAL_SECONDS", "60")) +WATCHDOG_CHECK_INTERVAL_SECONDS = int(os.getenv("WATCHDOG_CHECK_INTERVAL_SECONDS", "10")) + +# Configuration validation +def validate_config(): + """Validate required environment variables on startup.""" + required_vars = { + "NOCODB_URL": NOCODB_URL, + "NOCODB_TOKEN": NOCODB_TOKEN, + "RTMP_SERVER": RTMP_SERVER + } + + missing = [name for name, value in required_vars.items() if not value] + + if missing: + logger.error(f"Missing required environment variables: {', '.join(missing)}") + logger.error("Please set the following environment variables:") + logger.error(" NOCODB_URL - NocoDB API endpoint") + logger.error(" NOCODB_TOKEN - Authentication token") + logger.error(" RTMP_SERVER - Streaming destination") + sys.exit(1) + + # Validate directories exist or can be created + for dir_path in [RAW_DIR, FINAL_DIR]: + try: + dir_path.mkdir(parents=True, exist_ok=True) + except Exception as e: + logger.error(f"Cannot create directory {dir_path}: {e}") + sys.exit(1) + + logger.info("Configuration validated successfully") + logger.info(f"RAW_DIR: {RAW_DIR}") + logger.info(f"FINAL_DIR: {FINAL_DIR}") + logger.info(f"WHISPER_MODEL: {WHISPER_MODEL}") + logger.info(f"VAAPI_DEVICE: {VAAPI_DEVICE}") + logger.info(f"STREAM_GRACE_PERIOD: {STREAM_GRACE_PERIOD_MINUTES} minutes") + logger.info(f"NOCODB_SYNC_INTERVAL: {NOCODB_SYNC_INTERVAL_SECONDS} seconds") + logger.info(f"WATCHDOG_CHECK_INTERVAL: {WATCHDOG_CHECK_INTERVAL_SECONDS} seconds") + +# Database setup +db = sqlite3.connect("scheduler.db", check_same_thread=False) +db.row_factory = sqlite3.Row + +# Ensure table exists with log column and streaming metadata +db.execute(""" +CREATE TABLE IF NOT EXISTS jobs ( + nocodb_id TEXT PRIMARY KEY, + title TEXT, + run_at TIMESTAMP, + prep_at TIMESTAMP, + raw_path TEXT, + final_path TEXT, + prep_status TEXT, + play_status TEXT, + log TEXT, + stream_start_time TIMESTAMP, + stream_retry_count INTEGER DEFAULT 0 +) +""") +db.commit() + +# Add new columns if they don't exist (for existing databases) +try: + db.execute("ALTER TABLE jobs ADD COLUMN stream_start_time TIMESTAMP") + db.commit() +except: + pass + +try: + db.execute("ALTER TABLE jobs ADD COLUMN stream_retry_count INTEGER DEFAULT 0") + db.commit() +except: + pass + +# Track streaming process +streaming_process = None +current_streaming_job = None + +def log_to_db(nocodb_id, message): + """Append a timestamped log message to the database log column.""" + timestamp = datetime.now().isoformat() + log_entry = f"[{timestamp}] {message}\n" + + try: + db.execute(""" + UPDATE jobs SET log = log || ? WHERE nocodb_id = ? + """, (log_entry, nocodb_id)) + db.commit() + logger.info(f"Job {nocodb_id}: {message}") + except Exception as e: + logger.error(f"Failed to log to database: {e}") + +def retry_with_backoff(max_attempts=3, base_delay=1): + """Decorator to retry a function with exponential backoff.""" + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except Exception as e: + if attempt == max_attempts: + logger.error(f"{func.__name__} failed after {max_attempts} attempts: {e}") + raise + + delay = base_delay * (2 ** (attempt - 1)) + logger.warning(f"{func.__name__} attempt {attempt} failed: {e}. Retrying in {delay}s...") + time.sleep(delay) + + return wrapper + return decorator + +@retry_with_backoff(max_attempts=3) +def sync(): + """Sync jobs from NocoDB.""" + try: + response = requests.get( + NOCODB_URL, + headers={"xc-token": NOCODB_TOKEN}, + timeout=30 + ) + response.raise_for_status() + + rows = response.json().get("list", []) + logger.info(f"Fetched {len(rows)} jobs from NocoDB") + + for r in rows: + try: + run_at = datetime.fromisoformat(r["Date"]) + prep_at = run_at - timedelta(hours=6) + + # Preserve existing status and streaming data on sync + existing = db.execute("SELECT * FROM jobs WHERE nocodb_id=?", (r["Id"],)).fetchone() + if existing: + db.execute(""" + UPDATE jobs SET title=?, run_at=?, prep_at=? + WHERE nocodb_id=? + """, (r["Title"], run_at, prep_at, r["Id"])) + else: + db.execute(""" + INSERT INTO jobs (nocodb_id, title, run_at, prep_at, raw_path, final_path, + prep_status, play_status, log, stream_start_time, stream_retry_count) + VALUES (?,?,?,?,?,?,?,?,?,?,?) + """, (r["Id"], r["Title"], run_at, prep_at, None, None, 'pending', 'pending', '', None, 0)) + + except Exception as e: + logger.error(f"Failed to process row {r.get('Id', 'unknown')}: {e}") + + db.commit() + + except requests.exceptions.RequestException as e: + logger.error(f"Failed to sync from NocoDB: {e}") + raise + +def take_prep(): + """Get the next job ready for preparation.""" + try: + # First, mark jobs as skipped if both prep and run times have passed + db.execute(""" + UPDATE jobs + SET prep_status='skipped', play_status='skipped', + log = log || ? + WHERE prep_status='pending' + AND run_at <= CURRENT_TIMESTAMP + """, (f"[{datetime.now().isoformat()}] Job skipped - streaming time already passed\n",)) + db.commit() + + # Get next overdue or upcoming prep job + c = db.execute(""" + SELECT * FROM jobs WHERE prep_status='pending' AND prep_at <= CURRENT_TIMESTAMP LIMIT 1 + """) + job = c.fetchone() + + if job: + # Check if this is an overdue job + prep_at = datetime.fromisoformat(job["prep_at"]) + if prep_at < datetime.now() - timedelta(minutes=5): + logger.warning(f"Processing overdue prep job: {job['nocodb_id']} - {job['title']} (was due {prep_at})") + + return job + except Exception as e: + logger.error(f"Failed to query prep jobs: {e}") + return None + +def take_play(): + """Get the next job ready for streaming with configurable grace period.""" + try: + # Calculate grace period cutoff (default 15 minutes ago) + grace_period_cutoff = datetime.now() - timedelta(minutes=STREAM_GRACE_PERIOD_MINUTES) + + # Mark jobs more than STREAM_GRACE_PERIOD_MINUTES overdue as skipped + db.execute(""" + UPDATE jobs + SET play_status='skipped', + log = log || ? + WHERE prep_status='done' + AND play_status='pending' + AND run_at < ? + """, ( + f"[{datetime.now().isoformat()}] Streaming skipped - more than {STREAM_GRACE_PERIOD_MINUTES} minutes late\n", + grace_period_cutoff.isoformat() + )) + db.commit() + + # Get jobs ready to stream (on time or within 15-minute grace period) + c = db.execute(""" + SELECT * FROM jobs + WHERE prep_status='done' + AND play_status='pending' + AND run_at <= CURRENT_TIMESTAMP + LIMIT 1 + """) + job = c.fetchone() + + if job: + # Check if this is a late start + run_at = datetime.fromisoformat(job["run_at"]) + delay = datetime.now() - run_at + if delay > timedelta(seconds=30): + minutes_late = delay.total_seconds() / 60 + logger.warning(f"Starting stream late: {job['nocodb_id']} - {job['title']} ({minutes_late:.1f} minutes after scheduled time)") + + return job + except Exception as e: + logger.error(f"Failed to query play jobs: {e}") + return None + +def run_subprocess(cmd, job_id, description, timeout=None): + """Run a subprocess with error handling and logging.""" + log_to_db(job_id, f"Starting: {description}") + logger.info(f"Running command: {' '.join(str(c) for c in cmd)}") + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + check=True + ) + + if result.stdout: + logger.debug(f"stdout: {result.stdout[:500]}") + + log_to_db(job_id, f"Completed: {description}") + return result + + except subprocess.TimeoutExpired: + error_msg = f"Timeout: {description} exceeded {timeout}s" + log_to_db(job_id, f"ERROR: {error_msg}") + raise Exception(error_msg) + + except subprocess.CalledProcessError as e: + error_msg = f"Command failed with code {e.returncode}: {e.stderr[:500] if e.stderr else 'no error output'}" + log_to_db(job_id, f"ERROR: {description} - {error_msg}") + raise Exception(error_msg) + +@retry_with_backoff(max_attempts=3, base_delay=2) +def generate_subtitles(raw_file, job_id): + """Generate subtitles using whisper.cpp.""" + srt_file = raw_file.with_suffix(".srt") + + # Remove existing subtitle file if present + if srt_file.exists(): + srt_file.unlink() + log_to_db(job_id, f"Removed existing subtitle file: {srt_file}") + + # Run whisper.cpp with correct format: whisper.cpp -m -f -osrt + cmd = [ + "whisper.cpp", + "-m", str(WHISPER_MODEL), + "-f", str(raw_file), + "-osrt" + ] + + run_subprocess(cmd, job_id, "Subtitle generation with whisper.cpp", timeout=3600) + + # Verify subtitle file was created + if not srt_file.exists(): + error_msg = f"Subtitle file not created: {srt_file}" + log_to_db(job_id, f"ERROR: {error_msg}") + raise Exception(error_msg) + + log_to_db(job_id, f"Subtitle file created successfully: {srt_file}") + return srt_file + +@retry_with_backoff(max_attempts=3, base_delay=2) +def encode_video_with_subtitles(raw_file, srt_file, final_file, job_id): + """Encode video with burned-in subtitles using VAAPI.""" + # Remove existing output file if present + if final_file.exists(): + final_file.unlink() + log_to_db(job_id, f"Removed existing output file: {final_file}") + + # FFmpeg command with VAAPI encoding (h264_vaapi instead of h264_qsv) + cmd = [ + "ffmpeg", + "-hwaccel", "vaapi", + "-vaapi_device", VAAPI_DEVICE, + "-i", str(raw_file), + "-vf", f"subtitles={srt_file}:force_style=Fontname=Consolas,BackColour=&H80000000,Spacing=0.2,Outline=0,Shadow=0.75,format=yuv420p", + "-c:v", "h264_vaapi", # Changed from h264_qsv to h264_vaapi + "-qp", "23", # Quality parameter for VAAPI (similar to CRF) + "-c:a", "aac", + "-b:a", "192k", + "-movflags", "faststart", + "-y", # Overwrite output file + str(final_file) + ] + + run_subprocess(cmd, job_id, "Video encoding with VAAPI", timeout=7200) + + # Verify output file was created + if not final_file.exists(): + error_msg = f"Output video file not created: {final_file}" + log_to_db(job_id, f"ERROR: {error_msg}") + raise Exception(error_msg) + + file_size = final_file.stat().st_size / (1024 * 1024) # Size in MB + log_to_db(job_id, f"Video encoded successfully: {final_file} ({file_size:.2f} MB)") + return final_file + +def prepare_job(job): + """Prepare a job: generate subtitles and encode video.""" + job_id = job["nocodb_id"] + title = job["title"] + + try: + log_to_db(job_id, f"Starting preparation for: {title}") + + # Find raw video file + matching_files = list(RAW_DIR.glob(f"*{title}*")) + + if not matching_files: + error_msg = f"No files found matching title: {title}" + log_to_db(job_id, f"ERROR: {error_msg}") + db.execute("UPDATE jobs SET prep_status='failed' WHERE nocodb_id=?", (job_id,)) + db.commit() + return + + raw_file = max(matching_files, key=lambda x: x.stat().st_mtime) + log_to_db(job_id, f"Found raw file: {raw_file}") + + # Generate subtitles + srt_file = generate_subtitles(raw_file, job_id) + + # Prepare output filename + final_file = FINAL_DIR / (raw_file.stem + ".converted.mp4") + + # Encode video with subtitles + encode_video_with_subtitles(raw_file, srt_file, final_file, job_id) + + # Update database + db.execute(""" + UPDATE jobs SET prep_status='done', raw_path=?, final_path=? WHERE nocodb_id=? + """, (str(raw_file), str(final_file), job_id)) + db.commit() + + log_to_db(job_id, "Preparation completed successfully") + + except Exception as e: + error_msg = f"Preparation failed: {e}" + log_to_db(job_id, f"ERROR: {error_msg}") + logger.error(f"Job {job_id} preparation failed: {e}") + + db.execute("UPDATE jobs SET prep_status='failed' WHERE nocodb_id=?", (job_id,)) + db.commit() + +def get_video_duration(video_path): + """Get video duration in seconds using ffprobe.""" + try: + result = subprocess.run( + [ + "ffprobe", + "-v", "error", + "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", + str(video_path) + ], + capture_output=True, + text=True, + timeout=10 + ) + if result.returncode == 0 and result.stdout.strip(): + return float(result.stdout.strip()) + except Exception as e: + logger.error(f"Failed to get video duration: {e}") + return None + +def stream_job(job, seek_seconds=0): + """Start streaming a prepared job with optional seek position.""" + global streaming_process, current_streaming_job + + job_id = job["nocodb_id"] + final_path = job["final_path"] + + try: + # Check if starting late or restarting + run_at = datetime.fromisoformat(job["run_at"]) + delay = datetime.now() - run_at + if seek_seconds > 0: + log_to_db(job_id, f"Restarting stream at position {seek_seconds:.1f}s") + elif delay > timedelta(seconds=30): + minutes_late = delay.total_seconds() / 60 + log_to_db(job_id, f"Starting stream {minutes_late:.1f} minutes late") + + log_to_db(job_id, f"Starting stream to: {RTMP_SERVER}") + + # Verify file exists + if not Path(final_path).exists(): + error_msg = f"Final video file not found: {final_path}" + log_to_db(job_id, f"ERROR: {error_msg}") + db.execute("UPDATE jobs SET play_status='failed' WHERE nocodb_id=?", (job_id,)) + db.commit() + return False + + # Get video duration to know when it should finish + video_duration = get_video_duration(final_path) + if not video_duration: + logger.warning(f"Could not determine video duration for {final_path}") + + # Stop previous stream if running + if streaming_process and streaming_process.poll() is None: + logger.info(f"Stopping previous stream (PID: {streaming_process.pid})") + streaming_process.terminate() + try: + streaming_process.wait(timeout=5) + except subprocess.TimeoutExpired: + streaming_process.kill() + log_to_db(job_id, "Stopped previous stream") + + # Build ffmpeg command with optional seek + cmd = ["ffmpeg"] + + if seek_seconds > 0: + # Seek to position (input seeking is faster) + cmd.extend(["-ss", str(seek_seconds)]) + + cmd.extend([ + "-re", # Read input at native frame rate + "-i", final_path, + "-c", "copy", # Copy streams without re-encoding + "-f", "flv", + RTMP_SERVER + ]) + + logger.info(f"Starting stream: {' '.join(cmd)}") + streaming_process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + # Record when this stream started (for calculating seek position on restart) + stream_start_time = datetime.now() + current_streaming_job = job_id + + log_to_db(job_id, f"Stream started (PID: {streaming_process.pid})") + + # Update database - set status to 'streaming', not 'done' + db.execute(""" + UPDATE jobs + SET play_status='streaming', + stream_start_time=?, + stream_retry_count=stream_retry_count+1 + WHERE nocodb_id=? + """, (stream_start_time.isoformat(), job_id)) + db.commit() + + return True + + except Exception as e: + error_msg = f"Streaming failed: {e}" + log_to_db(job_id, f"ERROR: {error_msg}") + logger.error(f"Job {job_id} streaming failed: {e}") + + db.execute("UPDATE jobs SET play_status='failed' WHERE nocodb_id=?", (job_id,)) + db.commit() + return False + +def monitor_and_restart_stream(): + """Monitor active stream and restart if it fails.""" + global streaming_process, current_streaming_job + + # Check if there's supposed to be an active stream + try: + active_stream = db.execute(""" + SELECT * FROM jobs WHERE play_status='streaming' LIMIT 1 + """).fetchone() + + if not active_stream: + # No active stream expected + streaming_process = None + current_streaming_job = None + return + + job_id = active_stream["nocodb_id"] + final_path = active_stream["final_path"] + stream_start_time = active_stream["stream_start_time"] + retry_count = active_stream["stream_retry_count"] or 0 + + # Check if process is still running + if streaming_process and streaming_process.poll() is None: + # Stream is running fine + return + + # Stream is not running but should be + if streaming_process: + # Process exited - check if it was normal completion or error + return_code = streaming_process.returncode + if return_code == 0: + # Normal completion - video finished + logger.info(f"Stream completed successfully for job {job_id}") + log_to_db(job_id, "Stream completed successfully") + db.execute("UPDATE jobs SET play_status='done' WHERE nocodb_id=?", (job_id,)) + db.commit() + streaming_process = None + current_streaming_job = None + return + else: + # Error exit + stderr = streaming_process.stderr.read().decode('utf-8')[-500:] if streaming_process.stderr else "" + logger.error(f"Stream crashed for job {job_id} with code {return_code}: {stderr}") + log_to_db(job_id, f"Stream crashed (exit code {return_code})") + + # Calculate how much time has elapsed since stream started + if not stream_start_time: + logger.error(f"No stream start time recorded for job {job_id}") + db.execute("UPDATE jobs SET play_status='failed' WHERE nocodb_id=?", (job_id,)) + db.commit() + return + + start_time = datetime.fromisoformat(stream_start_time) + elapsed_seconds = (datetime.now() - start_time).total_seconds() + + # Get video duration to check if we should still be streaming + video_duration = get_video_duration(final_path) + if video_duration and elapsed_seconds >= video_duration: + # Video should have finished by now + logger.info(f"Stream duration exceeded for job {job_id} - marking as done") + log_to_db(job_id, "Stream duration completed") + db.execute("UPDATE jobs SET play_status='done' WHERE nocodb_id=?", (job_id,)) + db.commit() + streaming_process = None + current_streaming_job = None + return + + # Check retry limit (max 10 restarts) + if retry_count >= 10: + logger.error(f"Stream retry limit exceeded for job {job_id}") + log_to_db(job_id, "ERROR: Stream failed after 10 restart attempts") + db.execute("UPDATE jobs SET play_status='failed' WHERE nocodb_id=?", (job_id,)) + db.commit() + streaming_process = None + current_streaming_job = None + return + + # Restart stream at correct position + logger.warning(f"Restarting stream for job {job_id} at position {elapsed_seconds:.1f}s (attempt {retry_count + 1})") + stream_job(active_stream, seek_seconds=elapsed_seconds) + + except Exception as e: + logger.error(f"Error in stream monitor: {e}") + +def main(): + """Main scheduler loop.""" + logger.info("Starting scheduler...") + validate_config() + + # Check for overdue jobs on startup + try: + grace_period_cutoff = datetime.now() - timedelta(minutes=STREAM_GRACE_PERIOD_MINUTES) + + overdue_prep = db.execute(""" + SELECT COUNT(*) as count FROM jobs + WHERE prep_status='pending' AND prep_at <= CURRENT_TIMESTAMP AND run_at > CURRENT_TIMESTAMP + """).fetchone() + + skipped_prep = db.execute(""" + SELECT COUNT(*) as count FROM jobs + WHERE prep_status='pending' AND run_at <= CURRENT_TIMESTAMP + """).fetchone() + + overdue_stream = db.execute(""" + SELECT COUNT(*) as count FROM jobs + WHERE prep_status='done' AND play_status='pending' AND run_at <= CURRENT_TIMESTAMP AND run_at >= ? + """, (grace_period_cutoff.isoformat(),)).fetchone() + + skipped_stream = db.execute(""" + SELECT COUNT(*) as count FROM jobs + WHERE prep_status='done' AND play_status='pending' AND run_at < ? + """, (grace_period_cutoff.isoformat(),)).fetchone() + + if overdue_prep and overdue_prep["count"] > 0: + logger.warning(f"Found {overdue_prep['count']} overdue prep job(s) - will process immediately") + + if skipped_prep and skipped_prep["count"] > 0: + logger.warning(f"Found {skipped_prep['count']} unprepared job(s) past streaming time - will be marked as skipped") + + if overdue_stream and overdue_stream["count"] > 0: + logger.warning(f"Found {overdue_stream['count']} overdue streaming job(s) - will start within grace period ({STREAM_GRACE_PERIOD_MINUTES}min)") + + if skipped_stream and skipped_stream["count"] > 0: + logger.warning(f"Found {skipped_stream['count']} streaming job(s) more than {STREAM_GRACE_PERIOD_MINUTES}min late - will be marked as skipped") + + except Exception as e: + logger.error(f"Failed to check for overdue jobs: {e}") + + logger.info("Scheduler is running. Press Ctrl+C to stop.") + + try: + # Calculate how many iterations between syncs + sync_iterations = max(1, NOCODB_SYNC_INTERVAL_SECONDS // WATCHDOG_CHECK_INTERVAL_SECONDS) + logger.info(f"Main loop: checking every {WATCHDOG_CHECK_INTERVAL_SECONDS}s, syncing every {sync_iterations} iterations ({NOCODB_SYNC_INTERVAL_SECONDS}s)") + + iteration = 0 + while True: + try: + # Monitor and restart stream if needed (check every iteration) + monitor_and_restart_stream() + + # Sync jobs from NocoDB at configured interval + if iteration % sync_iterations == 0: + sync() + + # Process preparation jobs + job = take_prep() + if job: + logger.info(f"Processing prep job: {job['nocodb_id']} - {job['title']}") + prepare_job(job) + + # Process streaming jobs + job = take_play() + if job: + logger.info(f"Processing play job: {job['nocodb_id']} - {job['title']}") + stream_job(job) + + except Exception as e: + logger.error(f"Error in main loop: {e}") + + # Sleep between iterations + time.sleep(WATCHDOG_CHECK_INTERVAL_SECONDS) + iteration += 1 + + except KeyboardInterrupt: + logger.info("Scheduler stopped by user") + if streaming_process and streaming_process.poll() is None: + logger.info("Stopping active stream...") + streaming_process.terminate() + finally: + db.close() + +if __name__ == "__main__": + main() diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..29b4e3a --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,74 @@ +version: '3.8' + +services: + scheduler: + build: + context: . + dockerfile: Dockerfile + container_name: movie_scheduler + restart: unless-stopped + + # Environment variables - USE SECRETS IN PRODUCTION + env_file: + - .env.production + + # GPU access for VAAPI encoding + devices: + - /dev/dri:/dev/dri + group_add: + - video + - render + + volumes: + # Persistent database + - ./scheduler.db:/app/scheduler.db + + # Video storage - mount your actual storage paths + - /mnt/storage/raw_movies:/raw_movies:ro + - /mnt/storage/final_movies:/final_movies + + # Whisper model (download once, reuse) + - /opt/models:/models:ro + + # Logs (optional - use if not using external logging) + - ./logs:/app/logs + + # Logging configuration + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "5" + + # Resource limits + deploy: + resources: + limits: + cpus: '4.0' + memory: 4G + reservations: + cpus: '1.0' + memory: 1G + + # Health check + healthcheck: + test: ["CMD-SHELL", "pgrep -f agent.py || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + # Network + networks: + - scheduler_network + + # Security + security_opt: + - no-new-privileges:true + read_only: false + tmpfs: + - /tmp + +networks: + scheduler_network: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..20b0bdc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,112 @@ +version: '3.8' + +services: + # PostgreSQL database for NocoDB + postgres: + image: postgres:15-alpine + environment: + POSTGRES_DB: nocodb + POSTGRES_USER: nocodb + POSTGRES_PASSWORD: nocodb_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U nocodb"] + interval: 5s + timeout: 5s + retries: 5 + networks: + - scheduler_network + + # NocoDB service + nocodb: + image: nocodb/nocodb:latest + ports: + - "8080:8080" + environment: + NC_DB: "pg://postgres:5432?u=nocodb&p=nocodb_password&d=nocodb" + NC_AUTH_JWT_SECRET: "test_jwt_secret_key_12345" + NC_PUBLIC_URL: "http://localhost:8080" + depends_on: + postgres: + condition: service_healthy + volumes: + - nocodb_data:/usr/app/data + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/health || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + networks: + - scheduler_network + + # RTMP Server for streaming + rtmp: + image: tiangolo/nginx-rtmp + ports: + - "1935:1935" + - "8081:80" + volumes: + - ./nginx-rtmp.conf:/etc/nginx/nginx.conf:ro + - rtmp_recordings:/tmp/recordings + networks: + - scheduler_network + + # Initialization service - sets up test data + init: + build: + context: . + dockerfile: Dockerfile.init + depends_on: + nocodb: + condition: service_healthy + environment: + NOCODB_URL: "http://nocodb:8080" + NOCODB_EMAIL: "admin@test.com" + NOCODB_PASSWORD: "admin123" + volumes: + - raw_movies:/raw_movies + - config_data:/tmp + - ./init-data.sh:/init-data.sh:ro + networks: + - scheduler_network + command: ["/bin/bash", "/init-data.sh"] + + # Movie scheduler service + scheduler: + build: + context: . + dockerfile: Dockerfile + depends_on: + nocodb: + condition: service_healthy + rtmp: + condition: service_started + init: + condition: service_completed_successfully + environment: + RAW_DIR: "/raw_movies" + FINAL_DIR: "/final_movies" + WHISPER_MODEL: "/models/ggml-base.bin" + volumes: + - raw_movies:/raw_movies + - final_movies:/final_movies + - config_data:/config:ro + - ./scheduler.db:/app/scheduler.db + - ./start-scheduler.sh:/start-scheduler.sh:ro + networks: + - scheduler_network + restart: unless-stopped + command: ["/bin/bash", "/start-scheduler.sh"] + +volumes: + postgres_data: + nocodb_data: + raw_movies: + final_movies: + rtmp_recordings: + config_data: + +networks: + scheduler_network: + driver: bridge diff --git a/docker-helper.sh b/docker-helper.sh new file mode 100755 index 0000000..b0178f8 --- /dev/null +++ b/docker-helper.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# Docker Compose command detection +if docker compose version &> /dev/null; then + COMPOSE="docker compose" +elif docker-compose --version &> /dev/null; then + COMPOSE="docker-compose" +else + echo "Error: Docker Compose not found!" + exit 1 +fi + +case "$1" in + start) + echo "Starting all services..." + $COMPOSE up -d + ;; + stop) + echo "Stopping all services..." + $COMPOSE down + ;; + restart) + echo "Restarting all services..." + $COMPOSE restart + ;; + logs) + if [ -z "$2" ]; then + $COMPOSE logs -f + else + $COMPOSE logs -f "$2" + fi + ;; + status) + $COMPOSE ps + ;; + clean) + echo "Stopping and removing all containers and volumes..." + read -p "This will delete all data. Continue? (y/N) " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + $COMPOSE down -v + echo "Cleanup complete!" + else + echo "Cancelled." + fi + ;; + db) + echo "Opening scheduler database..." + if [ -f "scheduler.db" ]; then + sqlite3 scheduler.db + else + echo "Database file not found. Is the scheduler running?" + fi + ;; + jobs) + echo "Current jobs:" + if [ -f "scheduler.db" ]; then + sqlite3 scheduler.db "SELECT nocodb_id, title, prep_status, play_status, datetime(run_at) as scheduled FROM jobs ORDER BY run_at;" + else + echo "Database file not found." + fi + ;; + add-video) + if [ -z "$2" ]; then + echo "Usage: $0 add-video " + exit 1 + fi + if [ ! -f "$2" ]; then + echo "File not found: $2" + exit 1 + fi + echo "Adding video to raw_movies volume..." + docker run --rm -v scheduler_raw_movies:/data -v "$(pwd):/host" alpine \ + cp "/host/$2" /data/ + echo "✓ Video added: $(basename "$2")" + echo "Now add an entry in NocoDB with Title matching: $(basename "$2" | cut -d. -f1)" + ;; + stream) + echo "To watch the stream, use one of these commands:" + echo "" + echo "VLC:" + echo " vlc rtmp://localhost:1935/live/stream" + echo "" + echo "ffplay:" + echo " ffplay rtmp://localhost:1935/live/stream" + echo "" + echo "MPV:" + echo " mpv rtmp://localhost:1935/live/stream" + ;; + *) + echo "Movie Scheduler - Docker Helper" + echo "" + echo "Usage: $0 [arguments]" + echo "" + echo "Commands:" + echo " start Start all services" + echo " stop Stop all services" + echo " restart Restart all services" + echo " logs [service] View logs (all or specific service)" + echo " status Show service status" + echo " clean Stop and remove all data" + echo " db Open scheduler database (SQLite)" + echo " jobs List all jobs and their status" + echo " add-video Add video to raw_movies volume" + echo " stream Show commands to watch the stream" + echo "" + echo "Examples:" + echo " $0 start" + echo " $0 logs scheduler" + echo " $0 add-video my-movie.mp4" + echo " $0 jobs" + ;; +esac diff --git a/init-data.sh b/init-data.sh new file mode 100755 index 0000000..f848b38 --- /dev/null +++ b/init-data.sh @@ -0,0 +1,261 @@ +#!/bin/bash +set -e + +echo "========================================" +echo "NocoDB & Test Data Initialization" +echo "========================================" + +NOCODB_URL=${NOCODB_URL:-"http://nocodb:8080"} +NOCODB_EMAIL=${NOCODB_EMAIL:-"admin@test.com"} +NOCODB_PASSWORD=${NOCODB_PASSWORD:-"admin123"} + +# Wait for NocoDB to be fully ready +echo "Waiting for NocoDB to be ready..." +for i in {1..30}; do + if curl -sf "${NOCODB_URL}/api/v1/health" > /dev/null 2>&1; then + echo "NocoDB is ready!" + break + fi + echo "Attempt $i/30: NocoDB not ready yet, waiting..." + sleep 2 +done + +# Download test video (Big Buck Bunny - open source test video) +echo "" +echo "Downloading test video..." +if [ ! -f "/raw_movies/BigBuckBunny.mp4" ]; then + wget -q -O /raw_movies/BigBuckBunny.mp4 \ + "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4" \ + || echo "Failed to download test video, but continuing..." + + if [ -f "/raw_movies/BigBuckBunny.mp4" ]; then + echo "✓ Test video downloaded: BigBuckBunny.mp4" + else + echo "⚠ Could not download test video - you'll need to provide one manually" + fi +else + echo "✓ Test video already exists" +fi + +# Run Python script to setup NocoDB +echo "" +echo "Setting up NocoDB database and table..." +python3 << 'PYTHON_SCRIPT' +import requests +import json +import sys +import os +from datetime import datetime, timedelta + +NOCODB_URL = os.getenv("NOCODB_URL", "http://nocodb:8080") +EMAIL = os.getenv("NOCODB_EMAIL", "admin@test.com") +PASSWORD = os.getenv("NOCODB_PASSWORD", "admin123") + +print(f"Connecting to NocoDB at {NOCODB_URL}") + +# Step 1: Sign up (first time) or sign in +try: + # Try to sign up first + response = requests.post( + f"{NOCODB_URL}/api/v1/auth/user/signup", + json={ + "email": EMAIL, + "password": PASSWORD + }, + timeout=10 + ) + + if response.status_code == 200: + print("✓ New user created successfully") + auth_data = response.json() + else: + # If signup fails, try signin + response = requests.post( + f"{NOCODB_URL}/api/v1/auth/user/signin", + json={ + "email": EMAIL, + "password": PASSWORD + }, + timeout=10 + ) + response.raise_for_status() + print("✓ Signed in successfully") + auth_data = response.json() + +except Exception as e: + print(f"✗ Authentication failed: {e}") + sys.exit(1) + +token = auth_data.get("token") +if not token: + print("✗ No token received from NocoDB") + sys.exit(1) + +print(f"✓ Got authentication token") + +headers = { + "xc-auth": token, + "Content-Type": "application/json" +} + +# Step 2: Create a new base (project) +try: + response = requests.post( + f"{NOCODB_URL}/api/v1/db/meta/projects/", + headers=headers, + json={ + "title": "MovieScheduler", + "type": "database" + }, + timeout=10 + ) + + if response.status_code in [200, 201]: + base = response.json() + print(f"✓ Created base: {base.get('title')}") + else: + # Base might already exist, try to get it + response = requests.get( + f"{NOCODB_URL}/api/v1/db/meta/projects/", + headers=headers, + timeout=10 + ) + bases = response.json().get("list", []) + base = next((b for b in bases if b.get("title") == "MovieScheduler"), bases[0] if bases else None) + + if not base: + print("✗ Could not create or find base") + sys.exit(1) + print(f"✓ Using existing base: {base.get('title')}") + +except Exception as e: + print(f"✗ Base creation failed: {e}") + sys.exit(1) + +base_id = base.get("id") +print(f"Base ID: {base_id}") + +# Step 3: Get or create table +try: + # List existing tables + response = requests.get( + f"{NOCODB_URL}/api/v1/db/meta/projects/{base_id}/tables", + headers=headers, + timeout=10 + ) + tables = response.json().get("list", []) + + # Check if Movies table exists + movies_table = next((t for t in tables if t.get("title") == "Movies"), None) + + if not movies_table: + # Create Movies table + response = requests.post( + f"{NOCODB_URL}/api/v1/db/meta/projects/{base_id}/tables", + headers=headers, + json={ + "table_name": "Movies", + "title": "Movies", + "columns": [ + {"column_name": "Id", "title": "Id", "uidt": "ID"}, + {"column_name": "Title", "title": "Title", "uidt": "SingleLineText"}, + {"column_name": "Language", "title": "Language", "uidt": "SingleLineText"}, + {"column_name": "Year", "title": "Year", "uidt": "Year"}, + {"column_name": "Date", "title": "Date", "uidt": "DateTime"}, + {"column_name": "Image", "title": "Image", "uidt": "URL"}, + {"column_name": "IMDB", "title": "IMDB", "uidt": "URL"}, + {"column_name": "Time", "title": "Time", "uidt": "Duration"}, + {"column_name": "Kanal", "title": "Kanal", "uidt": "SingleLineText"}, + {"column_name": "Obejrzane", "title": "Obejrzane", "uidt": "Checkbox"} + ] + }, + timeout=10 + ) + response.raise_for_status() + movies_table = response.json() + print(f"✓ Created table: Movies") + else: + print(f"✓ Using existing table: Movies") + +except Exception as e: + print(f"✗ Table creation failed: {e}") + sys.exit(1) + +table_id = movies_table.get("id") +print(f"Table ID: {table_id}") + +# Step 4: Add test movie entries +test_movies = [ + { + "Title": "BigBuckBunny", + "Language": "po angielsku; z napisami", + "Year": 2008, + "Date": (datetime.now() + timedelta(minutes=5)).isoformat(), + "Image": "https://peach.blender.org/wp-content/uploads/title_anouncement.jpg", + "IMDB": "https://www.imdb.com/title/tt1254207/", + "Time": "9:56", + "Kanal": "TestTV", + "Obejrzane": False + }, + { + "Title": "TestMovie", + "Language": "po angielsku; z napisami", + "Year": 2024, + "Date": (datetime.now() + timedelta(hours=1)).isoformat(), + "Image": "https://example.com/test.jpg", + "IMDB": "https://www.imdb.com/", + "Time": "1:30:00", + "Kanal": "TestTV", + "Obejrzane": False + } +] + +try: + for movie in test_movies: + response = requests.post( + f"{NOCODB_URL}/api/v2/tables/{table_id}/records", + headers={"xc-token": token, "Content-Type": "application/json"}, + json=movie, + timeout=10 + ) + if response.status_code in [200, 201]: + print(f"✓ Added movie: {movie['Title']}") + else: + print(f"⚠ Could not add movie {movie['Title']}: {response.text}") + +except Exception as e: + print(f"⚠ Some movies could not be added: {e}") + +# Step 5: Save configuration for scheduler +print("\n" + "="*50) +print("CONFIGURATION FOR SCHEDULER") +print("="*50) +print(f"NOCODB_URL: http://nocodb:8080/api/v2/tables/{table_id}/records") +print(f"NOCODB_TOKEN: {token}") +print(f"RTMP_SERVER: rtmp://rtmp:1935/live/stream") +print("="*50) + +# Save to file that docker-compose can read +with open("/tmp/nocodb_config.env", "w") as f: + f.write(f"NOCODB_TABLE_ID={table_id}\n") + f.write(f"NOCODB_TOKEN={token}\n") + +print("\n✓ Configuration saved to /tmp/nocodb_config.env") +print("\n✓ Initialization complete!") + +PYTHON_SCRIPT + +echo "" +echo "========================================" +echo "✓ All initialization steps completed!" +echo "========================================" +echo "" +echo "NocoDB Dashboard: http://localhost:8080" +echo " Email: ${NOCODB_EMAIL}" +echo " Password: ${NOCODB_PASSWORD}" +echo "" +echo "RTMP Server: rtmp://localhost:1935/live/stream" +echo "RTMP Stats: http://localhost:8081/stat" +echo "" +echo "The scheduler will start automatically." +echo "========================================" diff --git a/nginx-rtmp.conf b/nginx-rtmp.conf new file mode 100644 index 0000000..38c6c17 --- /dev/null +++ b/nginx-rtmp.conf @@ -0,0 +1,66 @@ +worker_processes auto; +events { + worker_connections 1024; +} + +# RTMP configuration +rtmp { + server { + listen 1935; + chunk_size 4096; + allow publish all; + + application live { + live on; + record off; + + # Allow playback from anywhere + allow play all; + + # HLS settings (optional, for web playback) + hls on; + hls_path /tmp/hls; + hls_fragment 3; + hls_playlist_length 60; + + # Record streams (optional) + record all; + record_path /tmp/recordings; + record_suffix -%Y%m%d-%H%M%S.flv; + } + } +} + +# HTTP server for HLS playback +http { + server { + listen 80; + + # HLS streaming endpoint + location /hls { + types { + application/vnd.apple.mpegurl m3u8; + video/mp2t ts; + } + root /tmp; + add_header Cache-Control no-cache; + add_header Access-Control-Allow-Origin *; + } + + # RTMP statistics + location /stat { + rtmp_stat all; + rtmp_stat_stylesheet stat.xsl; + } + + location /stat.xsl { + root /usr/local/nginx/html; + } + + # Status page + location / { + root /usr/local/nginx/html; + index index.html; + } + } +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4a5625c --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requests>=2.25.0 diff --git a/scheduler.db b/scheduler.db new file mode 100644 index 0000000..38292db Binary files /dev/null and b/scheduler.db differ diff --git a/scheduler.service b/scheduler.service new file mode 100644 index 0000000..93958bb --- /dev/null +++ b/scheduler.service @@ -0,0 +1,56 @@ +[Unit] +Description=Movie Scheduler - Automated video processing and streaming +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=scheduler +Group=scheduler +WorkingDirectory=/opt/scheduler + +# Environment variables (or use EnvironmentFile) +Environment="NOCODB_URL=https://your-nocodb.com/api/v2/tables/YOUR_TABLE_ID/records" +Environment="NOCODB_TOKEN=YOUR_TOKEN_HERE" +Environment="RTMP_SERVER=rtmp://your-rtmp-server.com/live/stream" +Environment="RAW_DIR=/mnt/storage/raw_movies" +Environment="FINAL_DIR=/mnt/storage/final_movies" +Environment="WHISPER_MODEL=/opt/models/ggml-base.bin" +Environment="VAAPI_DEVICE=/dev/dri/renderD128" +Environment="STREAM_GRACE_PERIOD_MINUTES=15" +Environment="NOCODB_SYNC_INTERVAL_SECONDS=60" +Environment="WATCHDOG_CHECK_INTERVAL_SECONDS=10" + +# Or use external environment file (recommended for secrets) +# EnvironmentFile=/etc/scheduler/scheduler.env + +# Python virtual environment +ExecStart=/opt/scheduler/venv/bin/python /opt/scheduler/agent.py + +# Restart policy +Restart=always +RestartSec=10 + +# Security hardening +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/mnt/storage /opt/scheduler + +# Allow access to GPU for VAAPI +DeviceAllow=/dev/dri/renderD128 rw +SupplementaryGroups=video render + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=scheduler + +# Resource limits (adjust based on your needs) +LimitNOFILE=65536 +MemoryMax=4G +CPUQuota=200% + +[Install] +WantedBy=multi-user.target diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 0000000..fd56abc --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Scheduler Backup Script +# Backs up database and configuration with rotation +# + +set -e + +# Configuration +BACKUP_DIR="${BACKUP_DIR:-/backup/scheduler}" +DB_PATH="${DB_PATH:-/opt/scheduler/scheduler.db}" +CONFIG_PATH="${CONFIG_PATH:-/etc/scheduler/scheduler.env}" +RETENTION_DAYS=30 +COMPRESS_AFTER_DAYS=7 + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +DATE=$(date +%Y%m%d_%H%M%S) + +echo -e "${GREEN}=== Scheduler Backup - $(date) ===${NC}" + +# Create backup directory +mkdir -p "$BACKUP_DIR" + +# Check if database exists +if [ ! -f "$DB_PATH" ]; then + echo -e "${RED}✗ Database not found: $DB_PATH${NC}" + exit 1 +fi + +# Backup database +echo "Backing up database..." +cp "$DB_PATH" "$BACKUP_DIR/scheduler_${DATE}.db" +if [ $? -eq 0 ]; then + echo -e "${GREEN}✓ Database backed up: scheduler_${DATE}.db${NC}" +else + echo -e "${RED}✗ Database backup failed${NC}" + exit 1 +fi + +# Backup config if exists +if [ -f "$CONFIG_PATH" ]; then + echo "Backing up configuration..." + cp "$CONFIG_PATH" "$BACKUP_DIR/config_${DATE}.env" + if [ $? -eq 0 ]; then + echo -e "${GREEN}✓ Config backed up: config_${DATE}.env${NC}" + else + echo -e "${YELLOW}⚠ Config backup failed${NC}" + fi +fi + +# Get database stats +DB_SIZE=$(du -h "$BACKUP_DIR/scheduler_${DATE}.db" | cut -f1) +JOB_COUNT=$(sqlite3 "$BACKUP_DIR/scheduler_${DATE}.db" "SELECT COUNT(*) FROM jobs;" 2>/dev/null || echo "N/A") +echo -e "Database size: ${DB_SIZE}, Jobs: ${JOB_COUNT}" + +# Compress old backups (older than COMPRESS_AFTER_DAYS) +echo "Compressing old backups..." +COMPRESSED=$(find "$BACKUP_DIR" -name "*.db" -mtime +${COMPRESS_AFTER_DAYS} -exec gzip {} \; -print | wc -l) +if [ $COMPRESSED -gt 0 ]; then + echo -e "${GREEN}✓ Compressed $COMPRESSED old backup(s)${NC}" +fi + +# Delete backups older than RETENTION_DAYS +echo "Cleaning up old backups..." +DELETED_DB=$(find "$BACKUP_DIR" -name "*.db.gz" -mtime +${RETENTION_DAYS} -delete -print | wc -l) +DELETED_CFG=$(find "$BACKUP_DIR" -name "*.env" -mtime +${RETENTION_DAYS} -delete -print | wc -l) +if [ $DELETED_DB -gt 0 ] || [ $DELETED_CFG -gt 0 ]; then + echo -e "${GREEN}✓ Deleted $DELETED_DB database(s) and $DELETED_CFG config(s)${NC}" +fi + +# List recent backups +echo "" +echo "Recent backups:" +ls -lh "$BACKUP_DIR" | tail -5 + +# Optional: Upload to S3 or remote storage +# Uncomment and configure if needed +# +# if command -v aws &> /dev/null; then +# echo "Uploading to S3..." +# aws s3 cp "$BACKUP_DIR/scheduler_${DATE}.db" "s3://your-bucket/scheduler-backups/" +# echo -e "${GREEN}✓ Uploaded to S3${NC}" +# fi + +echo "" +echo -e "${GREEN}=== Backup completed ===${NC}" +echo "Backup location: $BACKUP_DIR/scheduler_${DATE}.db" diff --git a/scripts/monitor.sh b/scripts/monitor.sh new file mode 100755 index 0000000..ba03a18 --- /dev/null +++ b/scripts/monitor.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# +# Scheduler Monitoring Script +# Checks service health, job status, and system resources +# + +set -e + +# Configuration +LOG_FILE="${LOG_FILE:-/var/log/scheduler-monitor.log}" +DB_PATH="${DB_PATH:-/opt/scheduler/scheduler.db}" +SERVICE_NAME="${SERVICE_NAME:-scheduler}" +ALERT_EMAIL="${ALERT_EMAIL:-}" +DISK_THRESHOLD=90 + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S') +ALERTS=() + +echo "=== Scheduler Monitor - $TIMESTAMP ===" | tee -a "$LOG_FILE" + +# Check if service is running +check_service() { + if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then + echo -e "${GREEN}✓ Service is running${NC}" | tee -a "$LOG_FILE" + return 0 + elif docker ps --filter "name=movie_scheduler" --format "{{.Status}}" | grep -q "Up"; then + echo -e "${GREEN}✓ Docker container is running${NC}" | tee -a "$LOG_FILE" + return 0 + else + echo -e "${RED}✗ Service is DOWN${NC}" | tee -a "$LOG_FILE" + ALERTS+=("Service is DOWN") + + # Try to restart + echo "Attempting to restart service..." | tee -a "$LOG_FILE" + systemctl start "$SERVICE_NAME" 2>/dev/null || \ + docker compose -f /opt/scheduler/docker-compose.prod.yml restart 2>/dev/null || true + + return 1 + fi +} + +# Check database +check_database() { + if [ ! -f "$DB_PATH" ]; then + echo -e "${RED}✗ Database not found${NC}" | tee -a "$LOG_FILE" + ALERTS+=("Database file not found") + return 1 + fi + + # Database size + DB_SIZE=$(du -h "$DB_PATH" | cut -f1) + echo "Database size: $DB_SIZE" | tee -a "$LOG_FILE" + + # Job counts + TOTAL=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs;" 2>/dev/null) + PENDING_PREP=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE prep_status='pending';" 2>/dev/null) + PENDING_PLAY=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE play_status='pending';" 2>/dev/null) + STREAMING=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE play_status='streaming';" 2>/dev/null) + FAILED=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE prep_status='failed' OR play_status='failed';" 2>/dev/null) + + echo "Jobs - Total: $TOTAL, Pending prep: $PENDING_PREP, Pending play: $PENDING_PLAY, Streaming: $STREAMING, Failed: $FAILED" | tee -a "$LOG_FILE" + + if [ "$FAILED" -gt 0 ]; then + echo -e "${YELLOW}⚠ Found $FAILED failed job(s)${NC}" | tee -a "$LOG_FILE" + ALERTS+=("$FAILED failed jobs") + fi + + if [ "$STREAMING" -gt 0 ]; then + echo -e "${GREEN}✓ $STREAMING active stream(s)${NC}" | tee -a "$LOG_FILE" + fi +} + +# Check disk space +check_disk() { + for PATH in "/mnt/storage" "/opt/scheduler" "/"; do + if [ -d "$PATH" ]; then + USAGE=$(df "$PATH" 2>/dev/null | tail -1 | awk '{print $5}' | sed 's/%//') + if [ -n "$USAGE" ]; then + if [ "$USAGE" -gt "$DISK_THRESHOLD" ]; then + echo -e "${RED}✗ Disk usage for $PATH: ${USAGE}%${NC}" | tee -a "$LOG_FILE" + ALERTS+=("Disk usage ${USAGE}% on $PATH") + else + echo -e "Disk usage for $PATH: ${USAGE}%" | tee -a "$LOG_FILE" + fi + fi + fi + done +} + +# Check for stuck streams +check_stuck_streams() { + # Find streams that have been active for more than 4 hours + STUCK=$(sqlite3 "$DB_PATH" " + SELECT COUNT(*) FROM jobs + WHERE play_status='streaming' + AND datetime(stream_start_time) < datetime('now', '-4 hours') + " 2>/dev/null) + + if [ "$STUCK" -gt 0 ]; then + echo -e "${YELLOW}⚠ Found $STUCK stream(s) active for >4 hours${NC}" | tee -a "$LOG_FILE" + ALERTS+=("$STUCK potentially stuck streams") + fi +} + +# Check recent errors in logs +check_logs() { + if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then + ERROR_COUNT=$(journalctl -u "$SERVICE_NAME" --since "5 minutes ago" 2>/dev/null | grep -i "ERROR" | wc -l) + else + ERROR_COUNT=$(docker compose -f /opt/scheduler/docker-compose.prod.yml logs --since 5m 2>/dev/null | grep -i "ERROR" | wc -l || echo "0") + fi + + if [ "$ERROR_COUNT" -gt 10 ]; then + echo -e "${YELLOW}⚠ Found $ERROR_COUNT errors in last 5 minutes${NC}" | tee -a "$LOG_FILE" + ALERTS+=("$ERROR_COUNT recent errors") + fi +} + +# Send alerts +send_alerts() { + if [ ${#ALERTS[@]} -gt 0 ]; then + echo "" | tee -a "$LOG_FILE" + echo -e "${RED}=== ALERTS ===${NC}" | tee -a "$LOG_FILE" + for alert in "${ALERTS[@]}"; do + echo "- $alert" | tee -a "$LOG_FILE" + done + + # Send email if configured + if [ -n "$ALERT_EMAIL" ] && command -v mail &> /dev/null; then + { + echo "Scheduler Monitoring Alert - $TIMESTAMP" + echo "" + echo "The following issues were detected:" + for alert in "${ALERTS[@]}"; do + echo "- $alert" + done + echo "" + echo "Check $LOG_FILE for details" + } | mail -s "Scheduler Alert" "$ALERT_EMAIL" + fi + + # Could also send to Slack, PagerDuty, etc. + # Example Slack webhook: + # if [ -n "$SLACK_WEBHOOK" ]; then + # curl -X POST -H 'Content-type: application/json' \ + # --data "{\"text\":\"Scheduler Alert: ${ALERTS[*]}\"}" \ + # "$SLACK_WEBHOOK" + # fi + fi +} + +# Run checks +check_service +check_database +check_disk +check_stuck_streams +check_logs +send_alerts + +echo "" | tee -a "$LOG_FILE" + +# Exit with error if there are alerts +if [ ${#ALERTS[@]} -gt 0 ]; then + exit 1 +fi + +exit 0 diff --git a/scripts/setup-production.sh b/scripts/setup-production.sh new file mode 100755 index 0000000..c4900cf --- /dev/null +++ b/scripts/setup-production.sh @@ -0,0 +1,238 @@ +#!/bin/bash +# +# Production Setup Script +# Quick setup for production deployment +# + +set -e + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +echo -e "${BLUE}" +echo "==========================================" +echo " Movie Scheduler - Production Setup" +echo "==========================================" +echo -e "${NC}" + +# Check if running as root +if [ "$EUID" -eq 0 ]; then + echo -e "${RED}Please don't run as root. Run as your regular user.${NC}" + echo "The script will use sudo when needed." + exit 1 +fi + +# Detect deployment method +echo "Choose deployment method:" +echo "1) Systemd service (bare metal)" +echo "2) Docker container" +read -p "Enter choice [1-2]: " DEPLOY_METHOD + +if [ "$DEPLOY_METHOD" == "1" ]; then + echo -e "${GREEN}Setting up systemd service...${NC}" + + # Check dependencies + echo "Checking dependencies..." + for cmd in python3 ffmpeg git sqlite3 systemctl; do + if ! command -v $cmd &> /dev/null; then + echo -e "${RED}✗ Missing: $cmd${NC}" + echo "Install it with: sudo apt-get install $cmd" + exit 1 + else + echo -e "${GREEN}✓ Found: $cmd${NC}" + fi + done + + # Check whisper.cpp + if ! command -v whisper.cpp &> /dev/null; then + echo -e "${YELLOW}⚠ whisper.cpp not found${NC}" + read -p "Install whisper.cpp now? [y/N]: " INSTALL_WHISPER + if [[ $INSTALL_WHISPER =~ ^[Yy]$ ]]; then + echo "Installing whisper.cpp..." + git clone https://github.com/ggerganov/whisper.cpp.git /tmp/whisper.cpp + cd /tmp/whisper.cpp + make + sudo cp main /usr/local/bin/whisper.cpp + sudo chmod +x /usr/local/bin/whisper.cpp + rm -rf /tmp/whisper.cpp + echo -e "${GREEN}✓ whisper.cpp installed${NC}" + fi + fi + + # Create scheduler user + if ! id -u scheduler &> /dev/null; then + echo "Creating scheduler user..." + sudo useradd -r -s /bin/bash -d /opt/scheduler -m scheduler + sudo usermod -aG video,render scheduler + echo -e "${GREEN}✓ User created${NC}" + else + echo -e "${GREEN}✓ User already exists${NC}" + fi + + # Create directories + echo "Creating directories..." + sudo mkdir -p /opt/scheduler /opt/models + sudo mkdir -p /mnt/storage/raw_movies /mnt/storage/final_movies + sudo chown scheduler:scheduler /opt/scheduler + sudo chown scheduler:scheduler /mnt/storage/raw_movies /mnt/storage/final_movies + + # Copy application files + echo "Copying application files..." + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + + sudo -u scheduler cp "$PROJECT_DIR/agent.py" /opt/scheduler/ + sudo -u scheduler cp "$PROJECT_DIR/requirements.txt" /opt/scheduler/ + + # Create virtual environment + echo "Setting up Python virtual environment..." + sudo -u scheduler python3 -m venv /opt/scheduler/venv + sudo -u scheduler /opt/scheduler/venv/bin/pip install --upgrade pip + sudo -u scheduler /opt/scheduler/venv/bin/pip install -r /opt/scheduler/requirements.txt + + # Download whisper model if needed + if [ ! -f "/opt/models/ggml-base.bin" ]; then + echo "Downloading whisper model..." + sudo wget -q -O /opt/models/ggml-base.bin \ + https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin + echo -e "${GREEN}✓ Model downloaded${NC}" + fi + + # Create config directory + echo "Setting up configuration..." + sudo mkdir -p /etc/scheduler + + # Create environment file + if [ ! -f "/etc/scheduler/scheduler.env" ]; then + cat << 'EOF' | sudo tee /etc/scheduler/scheduler.env > /dev/null +# NocoDB Configuration +NOCODB_URL=https://your-nocodb.com/api/v2/tables/YOUR_TABLE_ID/records +NOCODB_TOKEN=YOUR_TOKEN_HERE + +# RTMP Server +RTMP_SERVER=rtmp://your-rtmp-server.com/live/stream + +# Storage Paths +RAW_DIR=/mnt/storage/raw_movies +FINAL_DIR=/mnt/storage/final_movies + +# Whisper Model +WHISPER_MODEL=/opt/models/ggml-base.bin + +# VAAPI Device +VAAPI_DEVICE=/dev/dri/renderD128 + +# Timing +NOCODB_SYNC_INTERVAL_SECONDS=60 +WATCHDOG_CHECK_INTERVAL_SECONDS=10 +STREAM_GRACE_PERIOD_MINUTES=15 +EOF + sudo chmod 600 /etc/scheduler/scheduler.env + sudo chown scheduler:scheduler /etc/scheduler/scheduler.env + + echo -e "${YELLOW}⚠ IMPORTANT: Edit /etc/scheduler/scheduler.env with your actual values${NC}" + echo " sudo nano /etc/scheduler/scheduler.env" + fi + + # Install systemd service + echo "Installing systemd service..." + sudo cp "$PROJECT_DIR/scheduler.service" /etc/systemd/system/ + sudo sed -i 's|^Environment=|# Environment=|g' /etc/systemd/system/scheduler.service + sudo sed -i 's|^# EnvironmentFile=|EnvironmentFile=|g' /etc/systemd/system/scheduler.service + sudo systemctl daemon-reload + + echo "" + echo -e "${GREEN}=== Installation Complete ===${NC}" + echo "" + echo "Next steps:" + echo "1. Edit configuration:" + echo " sudo nano /etc/scheduler/scheduler.env" + echo "" + echo "2. Enable and start service:" + echo " sudo systemctl enable scheduler" + echo " sudo systemctl start scheduler" + echo "" + echo "3. Check status:" + echo " sudo systemctl status scheduler" + echo "" + echo "4. View logs:" + echo " sudo journalctl -u scheduler -f" + +elif [ "$DEPLOY_METHOD" == "2" ]; then + echo -e "${GREEN}Setting up Docker deployment...${NC}" + + # Check docker + if ! command -v docker &> /dev/null; then + echo -e "${RED}✗ Docker not found${NC}" + echo "Install Docker first: https://docs.docker.com/get-docker/" + exit 1 + fi + + # Check docker compose + if ! docker compose version &> /dev/null && ! docker-compose --version &> /dev/null; then + echo -e "${RED}✗ Docker Compose not found${NC}" + exit 1 + fi + + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + + # Create deployment directory + read -p "Installation directory [/opt/scheduler]: " INSTALL_DIR + INSTALL_DIR=${INSTALL_DIR:-/opt/scheduler} + + sudo mkdir -p "$INSTALL_DIR" + sudo chown $USER:$USER "$INSTALL_DIR" + + # Copy files + echo "Copying files..." + cp "$PROJECT_DIR/agent.py" "$INSTALL_DIR/" + cp "$PROJECT_DIR/requirements.txt" "$INSTALL_DIR/" + cp "$PROJECT_DIR/Dockerfile" "$INSTALL_DIR/" + cp "$PROJECT_DIR/docker-compose.prod.yml" "$INSTALL_DIR/" + + # Create environment file + if [ ! -f "$INSTALL_DIR/.env.production" ]; then + cp "$PROJECT_DIR/.env.production.example" "$INSTALL_DIR/.env.production" + echo -e "${YELLOW}⚠ Edit $INSTALL_DIR/.env.production with your actual values${NC}" + fi + + # Create storage directories + sudo mkdir -p /mnt/storage/raw_movies /mnt/storage/final_movies + sudo mkdir -p /opt/models + + # Download whisper model + if [ ! -f "/opt/models/ggml-base.bin" ]; then + echo "Downloading whisper model..." + sudo wget -q -O /opt/models/ggml-base.bin \ + https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin + echo -e "${GREEN}✓ Model downloaded${NC}" + fi + + echo "" + echo -e "${GREEN}=== Installation Complete ===${NC}" + echo "" + echo "Next steps:" + echo "1. Edit configuration:" + echo " nano $INSTALL_DIR/.env.production" + echo "" + echo "2. Build and start:" + echo " cd $INSTALL_DIR" + echo " docker compose -f docker-compose.prod.yml up -d" + echo "" + echo "3. View logs:" + echo " docker compose -f docker-compose.prod.yml logs -f" + echo "" + echo "4. Check status:" + echo " docker compose -f docker-compose.prod.yml ps" + +else + echo -e "${RED}Invalid choice${NC}" + exit 1 +fi + +echo "" +echo -e "${BLUE}See PRODUCTION.md for complete documentation${NC}" diff --git a/start-scheduler.sh b/start-scheduler.sh new file mode 100755 index 0000000..9ac9b25 --- /dev/null +++ b/start-scheduler.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e + +echo "Starting Movie Scheduler..." + +# Wait for config file from init container +CONFIG_FILE="/config/nocodb_config.env" +echo "Waiting for initialization to complete..." + +for i in {1..60}; do + if [ -f "$CONFIG_FILE" ]; then + echo "✓ Configuration found!" + break + fi + if [ $i -eq 60 ]; then + echo "✗ Timeout waiting for configuration" + exit 1 + fi + sleep 1 +done + +# Load configuration +source "$CONFIG_FILE" + +# Set environment variables for the scheduler +export NOCODB_URL="http://nocodb:8080/api/v2/tables/${NOCODB_TABLE_ID}/records" +export NOCODB_TOKEN="${NOCODB_TOKEN}" +export RTMP_SERVER="${RTMP_SERVER:-rtmp://rtmp:1935/live/stream}" + +echo "Configuration loaded:" +echo " NOCODB_URL: $NOCODB_URL" +echo " RTMP_SERVER: $RTMP_SERVER" +echo "" +echo "Starting scheduler agent..." + +# Start the scheduler +exec python -u agent.py diff --git a/start-test-environment.sh b/start-test-environment.sh new file mode 100755 index 0000000..ef13609 --- /dev/null +++ b/start-test-environment.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +echo "==========================================" +echo " Movie Scheduler - Test Environment" +echo "==========================================" +echo "" + +# Make scripts executable +chmod +x init-data.sh start-scheduler.sh 2>/dev/null || true + +# Check if docker compose is available +if ! command -v docker &> /dev/null; then + echo "✗ Docker is not installed!" + echo "Please install Docker first: https://docs.docker.com/get-docker/" + exit 1 +fi + +if docker compose version &> /dev/null; then + COMPOSE_CMD="docker compose" +elif docker-compose --version &> /dev/null; then + COMPOSE_CMD="docker-compose" +else + echo "✗ Docker Compose is not installed!" + echo "Please install Docker Compose first" + exit 1 +fi + +echo "✓ Using: $COMPOSE_CMD" +echo "" + +# Stop any existing containers +echo "Stopping any existing containers..." +$COMPOSE_CMD down 2>/dev/null || true + +# Build and start all services +echo "" +echo "Building and starting all services..." +echo "This may take a few minutes on first run..." +echo "" + +$COMPOSE_CMD up --build -d + +echo "" +echo "==========================================" +echo " Waiting for services to initialize..." +echo "==========================================" +echo "" + +# Wait for init to complete +echo "Waiting for initialization to complete (this may take 30-60 seconds)..." +sleep 5 + +# Follow init logs +echo "" +echo "--- Initialization Progress ---" +$COMPOSE_CMD logs -f init & +LOGS_PID=$! + +# Wait for init to complete +while true; do + STATUS=$($COMPOSE_CMD ps init --format json 2>/dev/null | jq -r '.[0].State' 2>/dev/null || echo "unknown") + + if [ "$STATUS" = "exited" ]; then + kill $LOGS_PID 2>/dev/null || true + wait $LOGS_PID 2>/dev/null || true + echo "" + echo "✓ Initialization completed!" + break + fi + + sleep 2 +done + +echo "" +echo "==========================================" +echo " Test Environment Ready!" +echo "==========================================" +echo "" +echo "📺 NocoDB Dashboard:" +echo " URL: http://localhost:8080" +echo " Email: admin@test.com" +echo " Password: admin123" +echo "" +echo "📡 RTMP Server:" +echo " Stream URL: rtmp://localhost:1935/live/stream" +echo " Statistics: http://localhost:8081/stat" +echo "" +echo "🎬 Scheduler:" +echo " Status: Running" +echo " Database: ./scheduler.db" +echo "" +echo "==========================================" +echo " Useful Commands:" +echo "==========================================" +echo "" +echo "View scheduler logs:" +echo " $COMPOSE_CMD logs -f scheduler" +echo "" +echo "View all logs:" +echo " $COMPOSE_CMD logs -f" +echo "" +echo "Check service status:" +echo " $COMPOSE_CMD ps" +echo "" +echo "Stop all services:" +echo " $COMPOSE_CMD down" +echo "" +echo "Stop and remove all data:" +echo " $COMPOSE_CMD down -v" +echo "" +echo "==========================================" +echo "" +echo "The scheduler is now monitoring NocoDB for movies to process." +echo "A test movie 'BigBuckBunny' has been added and will be" +echo "processed 6 hours before its scheduled time." +echo "" +echo "To watch the stream, use a player like VLC:" +echo " vlc rtmp://localhost:1935/live/stream" +echo ""