initial commit - vibe coded with claude

This commit is contained in:
Maciej Bator 2026-01-21 14:44:17 +01:00
commit 7b826e9f38
19 changed files with 3761 additions and 0 deletions

44
Dockerfile Normal file
View File

@ -0,0 +1,44 @@
FROM python:3.11-slim
# Install system dependencies
RUN apt-get update && apt-get install -y \
ffmpeg \
wget \
git \
build-essential \
&& rm -rf /var/lib/apt/lists/*
# Install whisper.cpp
WORKDIR /tmp
RUN git clone https://github.com/ggerganov/whisper.cpp.git && \
cd whisper.cpp && \
make && \
cp main /usr/local/bin/whisper.cpp && \
chmod +x /usr/local/bin/whisper.cpp
# Download whisper model
RUN mkdir -p /models && \
cd /models && \
wget -q https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin && \
mv ggml-base.en.bin ggml-base.bin
# Set up application directory
WORKDIR /app
# Copy requirements and install Python dependencies
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Copy application code
COPY agent.py .
# Create directories for videos
RUN mkdir -p /raw_movies /final_movies
# Set environment variables with defaults
ENV RAW_DIR=/raw_movies \
FINAL_DIR=/final_movies \
WHISPER_MODEL=/models/ggml-base.bin \
VAAPI_DEVICE=/dev/dri/renderD128
CMD ["python", "-u", "agent.py"]

16
Dockerfile.init Normal file
View File

@ -0,0 +1,16 @@
FROM python:3.11-slim
# Install curl, wget and other utilities
RUN apt-get update && apt-get install -y \
curl \
wget \
jq \
&& rm -rf /var/lib/apt/lists/*
# Install Python requests library
RUN pip install --no-cache-dir requests
WORKDIR /app
# The init script will be mounted
CMD ["/bin/bash"]

875
PRODUCTION.md Normal file
View File

@ -0,0 +1,875 @@
# Production Deployment Guide
Complete guide for running the Movie Scheduler in production environments.
## Table of Contents
1. [Prerequisites](#prerequisites)
2. [Deployment Options](#deployment-options)
3. [Installation Methods](#installation-methods)
4. [Configuration](#configuration)
5. [Security](#security)
6. [Monitoring](#monitoring)
7. [Backup & Recovery](#backup--recovery)
8. [Maintenance](#maintenance)
9. [Troubleshooting](#troubleshooting)
---
## Prerequisites
### Hardware Requirements
**Minimum:**
- CPU: 4 cores (for whisper.cpp and encoding)
- RAM: 4GB
- Storage: 100GB+ (depends on video library size)
- GPU: Intel/AMD with VAAPI support (optional but recommended)
**Recommended:**
- CPU: 8+ cores
- RAM: 8GB+
- Storage: 500GB+ SSD
- GPU: Modern Intel/AMD GPU with VAAPI
### Software Requirements
- **OS**: Linux (Ubuntu 20.04+, Debian 11+, RHEL 8+, or compatible)
- **Python**: 3.7+
- **FFmpeg**: With VAAPI support
- **whisper.cpp**: Compiled and in PATH
- **Network**: Stable connection to NocoDB and RTMP server
---
## Deployment Options
### Option 1: Systemd Service (Recommended for bare metal)
✅ Direct hardware access (best VAAPI performance)
✅ Low overhead
✅ System integration
❌ Manual dependency management
### Option 2: Docker Container (Recommended for most users)
✅ Isolated environment
✅ Easy updates
✅ Portable configuration
⚠️ Slight performance overhead
⚠️ Requires GPU passthrough for VAAPI
### Option 3: Kubernetes/Orchestration
✅ High availability
✅ Auto-scaling
✅ Cloud-native
❌ Complex setup
❌ Overkill for single-instance deployment
---
## Installation Methods
### Method 1: Systemd Service Installation
#### 1. Create Scheduler User
```bash
# Create dedicated user for security
sudo useradd -r -s /bin/bash -d /opt/scheduler -m scheduler
# Add to video group for GPU access
sudo usermod -aG video,render scheduler
```
#### 2. Install Dependencies
```bash
# Install system packages
sudo apt-get update
sudo apt-get install -y python3 python3-pip python3-venv ffmpeg git build-essential
# Install whisper.cpp
sudo -u scheduler git clone https://github.com/ggerganov/whisper.cpp.git /tmp/whisper.cpp
cd /tmp/whisper.cpp
make
sudo cp main /usr/local/bin/whisper.cpp
sudo chmod +x /usr/local/bin/whisper.cpp
# Download whisper model
sudo mkdir -p /opt/models
cd /opt/models
sudo wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin
sudo mv ggml-base.en.bin ggml-base.bin
sudo chown -R scheduler:scheduler /opt/models
```
#### 3. Deploy Application
```bash
# Create application directory
sudo mkdir -p /opt/scheduler
sudo chown scheduler:scheduler /opt/scheduler
# Copy application files
sudo -u scheduler cp agent.py /opt/scheduler/
sudo -u scheduler cp requirements.txt /opt/scheduler/
# Create Python virtual environment
sudo -u scheduler python3 -m venv /opt/scheduler/venv
sudo -u scheduler /opt/scheduler/venv/bin/pip install -r /opt/scheduler/requirements.txt
```
#### 4. Configure Storage
```bash
# Create storage directories (adjust paths as needed)
sudo mkdir -p /mnt/storage/raw_movies
sudo mkdir -p /mnt/storage/final_movies
sudo chown -R scheduler:scheduler /mnt/storage
```
#### 5. Configure Service
```bash
# Copy service file
sudo cp scheduler.service /etc/systemd/system/
# Create environment file with secrets
sudo mkdir -p /etc/scheduler
sudo nano /etc/scheduler/scheduler.env
```
Edit `/etc/scheduler/scheduler.env`:
```bash
NOCODB_URL=https://your-nocodb.com/api/v2/tables/YOUR_TABLE_ID/records
NOCODB_TOKEN=your_production_token
RTMP_SERVER=rtmp://your-rtmp-server.com/live/stream
RAW_DIR=/mnt/storage/raw_movies
FINAL_DIR=/mnt/storage/final_movies
WHISPER_MODEL=/opt/models/ggml-base.bin
```
Update `scheduler.service` to use the environment file:
```ini
# Replace Environment= lines with:
EnvironmentFile=/etc/scheduler/scheduler.env
```
#### 6. Enable and Start Service
```bash
# Reload systemd
sudo systemctl daemon-reload
# Enable service (start on boot)
sudo systemctl enable scheduler
# Start service
sudo systemctl start scheduler
# Check status
sudo systemctl status scheduler
# View logs
sudo journalctl -u scheduler -f
```
---
### Method 2: Docker Deployment
#### 1. Prepare Environment
```bash
# Create project directory
mkdir -p /opt/scheduler
cd /opt/scheduler
# Copy application files
cp agent.py requirements.txt Dockerfile docker-compose.prod.yml ./
# Create production environment file
cp .env.production.example .env.production
nano .env.production # Fill in your values
```
#### 2. Configure Storage
```bash
# Ensure storage directories exist
mkdir -p /mnt/storage/raw_movies
mkdir -p /mnt/storage/final_movies
# Download whisper model
mkdir -p /opt/models
cd /opt/models
wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin
mv ggml-base.en.bin ggml-base.bin
```
#### 3. Deploy Container
```bash
cd /opt/scheduler
# Build image
docker compose -f docker-compose.prod.yml build
# Start service
docker compose -f docker-compose.prod.yml up -d
# Check logs
docker compose -f docker-compose.prod.yml logs -f
# Check status
docker compose -f docker-compose.prod.yml ps
```
#### 4. Enable Auto-Start
```bash
# Create systemd service for docker compose
sudo nano /etc/systemd/system/scheduler-docker.service
```
```ini
[Unit]
Description=Movie Scheduler (Docker)
Requires=docker.service
After=docker.service
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=/opt/scheduler
ExecStart=/usr/bin/docker compose -f docker-compose.prod.yml up -d
ExecStop=/usr/bin/docker compose -f docker-compose.prod.yml down
TimeoutStartSec=0
[Install]
WantedBy=multi-user.target
```
```bash
sudo systemctl daemon-reload
sudo systemctl enable scheduler-docker
```
---
## Configuration
### Essential Configuration
#### NocoDB Connection
```bash
# Get your table ID from NocoDB URL
# https://nocodb.com/nc/YOUR_BASE_ID/table_NAME
# API endpoint: https://nocodb.com/api/v2/tables/TABLE_ID/records
# Generate API token in NocoDB:
# Account Settings → Tokens → Create Token
```
#### RTMP Server
```bash
# For nginx-rtmp:
RTMP_SERVER=rtmp://your-server.com:1935/live/stream
# For other RTMP servers, use their endpoint format
```
#### Storage Paths
```bash
# Use separate fast storage for final videos (streaming)
RAW_DIR=/mnt/storage/raw_movies # Can be slower storage
FINAL_DIR=/mnt/fast-storage/final # Should be fast SSD
# Ensure proper permissions
chown -R scheduler:scheduler /mnt/storage
chmod 755 /mnt/storage/raw_movies
chmod 755 /mnt/fast-storage/final
```
### Performance Tuning
#### Sync Intervals
```bash
# High-load scenario (many jobs, frequent updates)
NOCODB_SYNC_INTERVAL_SECONDS=120 # Check less often
WATCHDOG_CHECK_INTERVAL_SECONDS=15 # Check streams less often
# Low-latency scenario (need fast response)
NOCODB_SYNC_INTERVAL_SECONDS=30
WATCHDOG_CHECK_INTERVAL_SECONDS=5
# Default (balanced)
NOCODB_SYNC_INTERVAL_SECONDS=60
WATCHDOG_CHECK_INTERVAL_SECONDS=10
```
#### FFmpeg VAAPI
```bash
# Find your VAAPI device
ls -la /dev/dri/
# Common devices:
# renderD128 - Primary GPU
# renderD129 - Secondary GPU
# Test VAAPI
ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -i test.mp4 -f null -
# Set in config
VAAPI_DEVICE=/dev/dri/renderD128
```
---
## Security
### Secrets Management
**DO NOT hardcode secrets in files tracked by git!**
#### Option 1: Environment Files (Simple)
```bash
# Store secrets in protected file
sudo nano /etc/scheduler/scheduler.env
sudo chmod 600 /etc/scheduler/scheduler.env
sudo chown scheduler:scheduler /etc/scheduler/scheduler.env
```
#### Option 2: Secrets Management Tools
```bash
# Using Vault
export NOCODB_TOKEN=$(vault kv get -field=token secret/scheduler/nocodb)
# Using AWS Secrets Manager
export NOCODB_TOKEN=$(aws secretsmanager get-secret-value --secret-id scheduler/nocodb --query SecretString --output text)
# Using Docker Secrets (Swarm/Kubernetes)
# Mount secrets as files and read in application
```
### Filesystem Permissions
```bash
# Application directory
chown -R scheduler:scheduler /opt/scheduler
chmod 750 /opt/scheduler
# Storage directories
chown -R scheduler:scheduler /mnt/storage
chmod 755 /mnt/storage/raw_movies # Read-only for scheduler
chmod 755 /mnt/storage/final_movies # Read-write for scheduler
# Database file
chmod 600 /opt/scheduler/scheduler.db
chown scheduler:scheduler /opt/scheduler/scheduler.db
```
### Network Security
```bash
# Firewall rules (if scheduler runs on separate server)
# Only allow outbound connections to NocoDB and RTMP
sudo ufw allow out to YOUR_NOCODB_IP port 443 # HTTPS
sudo ufw allow out to YOUR_RTMP_IP port 1935 # RTMP
sudo ufw default deny outgoing # Deny all other outbound (optional)
```
### Regular Updates
```bash
# Update system packages weekly
sudo apt-get update && sudo apt-get upgrade
# Update Python dependencies
sudo -u scheduler /opt/scheduler/venv/bin/pip install --upgrade requests
# Rebuild whisper.cpp quarterly (for performance improvements)
```
---
## Monitoring
### Service Health
#### Systemd Monitoring
```bash
# Check service status
systemctl status scheduler
# View recent logs
journalctl -u scheduler -n 100
# Follow logs in real-time
journalctl -u scheduler -f
# Check for errors in last hour
journalctl -u scheduler --since "1 hour ago" | grep ERROR
# Service restart count
systemctl show scheduler | grep NRestarts
```
#### Docker Monitoring
```bash
# Container status
docker compose -f docker-compose.prod.yml ps
# Resource usage
docker stats movie_scheduler
# Logs
docker compose -f docker-compose.prod.yml logs --tail=100 -f
# Health check status
docker inspect movie_scheduler | jq '.[0].State.Health'
```
### Database Monitoring
```bash
# Check job status
sqlite3 /opt/scheduler/scheduler.db "SELECT prep_status, play_status, COUNT(*) FROM jobs GROUP BY prep_status, play_status;"
# Active streams
sqlite3 /opt/scheduler/scheduler.db "SELECT nocodb_id, title, play_status, stream_retry_count FROM jobs WHERE play_status='streaming';"
# Failed jobs
sqlite3 /opt/scheduler/scheduler.db "SELECT nocodb_id, title, prep_status, play_status, log FROM jobs WHERE prep_status='failed' OR play_status='failed';"
# Database size
ls -lh /opt/scheduler/scheduler.db
```
### Automated Monitoring Script
Create `/opt/scheduler/monitor.sh`:
```bash
#!/bin/bash
LOG_FILE="/var/log/scheduler-monitor.log"
DB_PATH="/opt/scheduler/scheduler.db"
echo "=== Scheduler Monitor - $(date) ===" >> "$LOG_FILE"
# Check if service is running
if systemctl is-active --quiet scheduler; then
echo "✓ Service is running" >> "$LOG_FILE"
else
echo "✗ Service is DOWN" >> "$LOG_FILE"
# Send alert (email, Slack, etc.)
systemctl start scheduler
fi
# Check for failed jobs
FAILED=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE prep_status='failed' OR play_status='failed';")
if [ "$FAILED" -gt 0 ]; then
echo "⚠ Found $FAILED failed jobs" >> "$LOG_FILE"
# Send alert
fi
# Check active streams
STREAMING=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE play_status='streaming';")
echo "Active streams: $STREAMING" >> "$LOG_FILE"
# Check disk space
DISK_USAGE=$(df -h /mnt/storage | tail -1 | awk '{print $5}' | sed 's/%//')
if [ "$DISK_USAGE" -gt 90 ]; then
echo "⚠ Disk usage is ${DISK_USAGE}%" >> "$LOG_FILE"
# Send alert
fi
echo "" >> "$LOG_FILE"
```
```bash
# Make executable
chmod +x /opt/scheduler/monitor.sh
# Add to crontab (check every 5 minutes)
(crontab -l 2>/dev/null; echo "*/5 * * * * /opt/scheduler/monitor.sh") | crontab -
```
### External Monitoring
#### Prometheus + Grafana
Export metrics using node_exporter or custom exporter:
```bash
# Install node_exporter for system metrics
# Create custom exporter for job metrics from database
# Set up Grafana dashboard
```
#### Uptime Monitoring
Use services like:
- UptimeRobot
- Pingdom
- Datadog
Monitor:
- Service availability
- RTMP server connectivity
- NocoDB API accessibility
---
## Backup & Recovery
### What to Backup
1. **Database** (scheduler.db) - Critical
2. **Configuration** (.env.production or /etc/scheduler/scheduler.env) - Critical
3. **Final videos** (if you want to keep processed videos)
4. **Logs** (optional, for forensics)
### Backup Script
Create `/opt/scheduler/backup.sh`:
```bash
#!/bin/bash
BACKUP_DIR="/backup/scheduler"
DATE=$(date +%Y%m%d_%H%M%S)
DB_PATH="/opt/scheduler/scheduler.db"
CONFIG_PATH="/etc/scheduler/scheduler.env"
# Create backup directory
mkdir -p "$BACKUP_DIR"
# Backup database
cp "$DB_PATH" "$BACKUP_DIR/scheduler_${DATE}.db"
# Backup config (careful with secrets!)
cp "$CONFIG_PATH" "$BACKUP_DIR/config_${DATE}.env"
# Compress old backups
find "$BACKUP_DIR" -name "*.db" -mtime +7 -exec gzip {} \;
# Delete backups older than 30 days
find "$BACKUP_DIR" -name "*.db.gz" -mtime +30 -delete
find "$BACKUP_DIR" -name "*.env" -mtime +30 -delete
# Optional: Upload to S3/cloud storage
# aws s3 sync "$BACKUP_DIR" s3://your-bucket/scheduler-backups/
echo "Backup completed: $BACKUP_DIR/scheduler_${DATE}.db"
```
```bash
# Make executable
chmod +x /opt/scheduler/backup.sh
# Run daily at 2 AM
(crontab -l 2>/dev/null; echo "0 2 * * * /opt/scheduler/backup.sh") | crontab -
```
### Recovery Procedure
#### 1. Restore from Backup
```bash
# Stop service
sudo systemctl stop scheduler
# Restore database
cp /backup/scheduler/scheduler_YYYYMMDD_HHMMSS.db /opt/scheduler/scheduler.db
chown scheduler:scheduler /opt/scheduler/scheduler.db
# Restore config if needed
cp /backup/scheduler/config_YYYYMMDD_HHMMSS.env /etc/scheduler/scheduler.env
# Start service
sudo systemctl start scheduler
```
#### 2. Disaster Recovery (Full Rebuild)
If server is completely lost:
1. Provision new server
2. Follow installation steps above
3. Restore database and config from backup
4. Restart service
5. Verify jobs are picked up
**Recovery Time Objective (RTO):** 30-60 minutes
**Recovery Point Objective (RPO):** Up to 24 hours (with daily backups)
---
## Maintenance
### Routine Tasks
#### Daily
- ✓ Check service status
- ✓ Review error logs
- ✓ Check failed jobs
#### Weekly
- ✓ Review disk space
- ✓ Check database size
- ✓ Clean up old processed videos (if not needed)
#### Monthly
- ✓ Update system packages
- ✓ Review and optimize database
- ✓ Test backup restoration
- ✓ Review and rotate logs
### Database Maintenance
```bash
# Vacuum database (reclaim space, optimize)
sqlite3 /opt/scheduler/scheduler.db "VACUUM;"
# Analyze database (update statistics)
sqlite3 /opt/scheduler/scheduler.db "ANALYZE;"
# Clean up old completed jobs (optional)
sqlite3 /opt/scheduler/scheduler.db "DELETE FROM jobs WHERE play_status='done' AND datetime(run_at) < datetime('now', '-30 days');"
```
### Log Rotation
For systemd (automatic via journald):
```bash
# Configure in /etc/systemd/journald.conf
SystemMaxUse=1G
RuntimeMaxUse=100M
```
For Docker:
```yaml
# Already configured in docker-compose.prod.yml
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "5"
```
### Video Cleanup
```bash
# Clean up old final videos (adjust retention as needed)
find /mnt/storage/final_movies -name "*.mp4" -mtime +7 -delete
# Or move to archive
find /mnt/storage/final_movies -name "*.mp4" -mtime +7 -exec mv {} /mnt/archive/ \;
```
---
## Troubleshooting
### Service Won't Start
```bash
# Check service status
systemctl status scheduler
# Check logs for errors
journalctl -u scheduler -n 50
# Common issues:
# 1. Missing environment variables
grep -i "error" /var/log/syslog | grep scheduler
# 2. Permission issues
ls -la /opt/scheduler
ls -la /mnt/storage
# 3. GPU access issues
ls -la /dev/dri/
groups scheduler # Should include 'video' and 'render'
```
### Streams Keep Failing
```bash
# Test RTMP server manually
ffmpeg -re -i test.mp4 -c copy -f flv rtmp://your-server/live/stream
# Check network connectivity
ping your-rtmp-server.com
telnet your-rtmp-server.com 1935
# Review stream logs
journalctl -u scheduler | grep -A 10 "Stream crashed"
# Check retry count
sqlite3 /opt/scheduler/scheduler.db "SELECT nocodb_id, title, stream_retry_count FROM jobs WHERE play_status='streaming';"
```
### High CPU/Memory Usage
```bash
# Check resource usage
top -u scheduler
# Or for Docker
docker stats movie_scheduler
# Common causes:
# 1. Large video file encoding - normal, wait for completion
# 2. whisper.cpp using all cores - normal
# 3. Multiple prep jobs running - adjust or wait
# Limit resources if needed (systemd)
systemctl edit scheduler
# Add:
[Service]
CPUQuota=200%
MemoryMax=4G
```
### Database Locked Errors
```bash
# Check for stale locks
lsof /opt/scheduler/scheduler.db
# Kill stale processes if needed
# Restart service
systemctl restart scheduler
```
### VAAPI Not Working
```bash
# Verify VAAPI support
vainfo
# Test FFmpeg VAAPI
ffmpeg -hwaccels
# Check permissions
ls -la /dev/dri/renderD128
groups scheduler # Should include 'video' or 'render'
# Fallback to software encoding
# Comment out VAAPI_DEVICE in config
# Encoding will use CPU (slower but works)
```
---
## Performance Optimization
### Hardware Acceleration
```bash
# Verify GPU usage during encoding
intel_gpu_top # For Intel GPUs
radeontop # For AMD GPUs
# If GPU not being used, check:
# 1. VAAPI device path correct
# 2. User has GPU permissions
# 3. FFmpeg compiled with VAAPI support
```
### Storage Performance
```bash
# Use SSD for final videos (they're streamed frequently)
# Use HDD for raw videos (accessed once for processing)
# Test disk performance
dd if=/dev/zero of=/mnt/storage/test bs=1M count=1024 oflag=direct
rm /mnt/storage/test
```
### Network Optimization
```bash
# For better streaming reliability
# 1. Use dedicated network for RTMP
# 2. Enable QoS for streaming traffic
# 3. Consider local RTMP relay
# Test network throughput
iperf3 -c your-rtmp-server.com
```
---
## Production Checklist
Before going live:
- [ ] Secrets stored securely (not in git)
- [ ] Service auto-starts on boot
- [ ] Backups configured and tested
- [ ] Monitoring configured
- [ ] Logs being rotated
- [ ] Disk space alerts configured
- [ ] Test recovery procedure
- [ ] Document runbook for on-call
- [ ] GPU permissions verified
- [ ] RTMP connectivity tested
- [ ] NocoDB API tested
- [ ] Process one test video end-to-end
- [ ] Verify streaming watchdog works
- [ ] Test service restart during streaming
- [ ] Configure alerting for failures
---
## Support & Updates
### Getting Updates
```bash
# Git-based deployment
cd /opt/scheduler
git pull origin main
systemctl restart scheduler
# Docker-based deployment
cd /opt/scheduler
docker compose -f docker-compose.prod.yml pull
docker compose -f docker-compose.prod.yml up -d
```
### Reporting Issues
Include in bug reports:
- Service logs: `journalctl -u scheduler -n 100`
- Database state: `sqlite3 scheduler.db ".dump jobs"`
- System info: `uname -a`, `python3 --version`, `ffmpeg -version`
- Configuration (redact secrets!)
---
## Additional Resources
- FFmpeg VAAPI Guide: https://trac.ffmpeg.org/wiki/Hardware/VAAPI
- whisper.cpp: https://github.com/ggerganov/whisper.cpp
- NocoDB API: https://docs.nocodb.com
- Systemd Documentation: https://www.freedesktop.org/software/systemd/man/
---
## License
This production guide is provided as-is. Test thoroughly in staging before production deployment.

314
README.md Normal file
View File

@ -0,0 +1,314 @@
# Movie Scheduler
Automated movie scheduler that reads from NocoDB, generates subtitles with whisper.cpp, burns them into videos with VAAPI encoding, and streams at scheduled times.
## Quick Links
- **[Production Deployment Guide](PRODUCTION.md)** - Complete guide for production setup
- **[Testing Guide](TESTING.md)** - Docker-based test environment
- **[Development Setup](#installation)** - Local development setup
## Features
- **NocoDB Integration**: Syncs job schedules from NocoDB database
- **Automatic Subtitle Generation**: Uses whisper.cpp to generate SRT subtitles
- **Hardware-Accelerated Encoding**: Burns subtitles into videos using VAAPI (h264_vaapi)
- **Scheduled Streaming**: Streams videos to RTMP server at scheduled times
- **Robust Error Handling**: Automatic retries with exponential backoff
- **Comprehensive Logging**: Tracks all operations in database log column
- **Process Management**: Properly tracks and manages streaming processes
## Requirements
- Python 3.7+
- `requests` library
- `whisper.cpp` installed and available in PATH
- `ffmpeg` with VAAPI support
- VAAPI-capable hardware (AMD/Intel GPU)
- NocoDB instance with scheduled jobs table
## Installation
1. Clone or download this repository
2. Install Python dependencies:
```bash
pip install requests
```
3. Ensure whisper.cpp is installed:
```bash
# Follow whisper.cpp installation instructions
# Make sure the binary is in your PATH
which whisper.cpp
```
4. Verify FFmpeg has VAAPI support:
```bash
ffmpeg -hwaccels | grep vaapi
```
## Configuration
1. Copy the example environment file:
```bash
cp .env.example .env
```
2. Edit `.env` and set the required variables:
```bash
# Required
export NOCODB_URL="https://nocodb/api/v2/tables/XXXX/records"
export NOCODB_TOKEN="your_token_here"
export RTMP_SERVER="rtmp://your_server/live"
# Optional (defaults provided)
export RAW_DIR="/root/surowe_filmy"
export FINAL_DIR="/root/przygotowane_filmy"
export WHISPER_MODEL="/root/models/ggml-base.bin"
export VAAPI_DEVICE="/dev/dri/renderD128"
export STREAM_GRACE_PERIOD_MINUTES="15"
export NOCODB_SYNC_INTERVAL_SECONDS="60"
export WATCHDOG_CHECK_INTERVAL_SECONDS="10"
```
3. Load the environment variables:
```bash
source .env
```
## NocoDB Table Structure
Your NocoDB table should have the following fields:
- `Id` - Unique identifier (Text/Primary Key)
- `Title` - Movie title matching filename (Text)
- `Date` - Scheduled run time (DateTime in ISO format)
The scheduler will automatically create additional columns in the local SQLite database:
- `prep_at` - Preparation time (6 hours before run_at)
- `prep_status` - Preparation status (pending/done/failed)
- `play_status` - Streaming status (pending/done/failed)
- `raw_path` - Path to source video file
- `final_path` - Path to converted video with subtitles
- `log` - Detailed operation log
## Usage
### Running the Scheduler
```bash
# Ensure environment variables are set
source .env
# Run the scheduler
python agent.py
```
The scheduler will:
1. Sync jobs from NocoDB every 60 seconds (configurable)
2. Check stream health every 10 seconds (configurable)
3. Prepare videos 6 hours before their scheduled time:
- Find matching video file in RAW_DIR
- Generate subtitles with whisper.cpp
- Encode video with burned subtitles using VAAPI
4. Stream videos at their scheduled time to RTMP server
### Stopping the Scheduler
Press `Ctrl+C` to gracefully stop the scheduler. It will:
- Stop any active streaming process
- Close the database connection
- Exit cleanly
### Restart & Recovery Behavior
The scheduler handles restarts, power outages, and downtime gracefully:
**On Startup:**
- ✅ Checks for overdue prep jobs and processes them immediately
- ⚠️ Skips jobs where the streaming time has already passed
- 📊 Logs recovery status (overdue/skipped jobs found)
**Grace Period for Late Streaming:**
- ⏰ If prep is done and streaming time is overdue by **up to 15 minutes**, will still start streaming
- ⚠️ Jobs more than 15 minutes late are marked as 'skipped'
- 📝 Late starts are logged with exact delay time
**Recovery Scenarios:**
1. **Short Outage (prep time missed, but streaming time not reached)**
- Movie scheduled for 21:00 (prep at 15:00)
- System down from 14:00-16:00
- ✅ Restarts at 16:00: immediately preps the movie
- ✅ Streams normally at 21:00
2. **Late Start Within Grace Period (< 15 minutes)**
- Movie scheduled for 21:00 (prep completed at 15:00)
- System down from 20:00-21:10
- ✅ Restarts at 21:10: starts streaming immediately (10 minutes late)
- 📝 Logged: "Starting stream 10.0 minutes late"
3. **Too Late to Stream (> 15 minutes)**
- Movie scheduled for 21:00 (prep completed)
- System down from 20:00-21:20
- ⚠️ Restarts at 21:20: marks streaming as 'skipped' (>15 min late)
- 📝 Logged: "Streaming skipped - more than 15 minutes late"
4. **Long Outage (both prep and streaming times passed, >15 min)**
- Movie scheduled for 21:00 (prep at 15:00)
- System down from 14:00-22:00
- ⚠️ Restarts at 22:00: marks entire job as 'skipped'
- 📝 Logged: "Job skipped - streaming time already passed"
5. **Crash During Processing**
- System crashes during subtitle generation or encoding
- ✅ On restart: retries from the beginning with full retry logic
- All operations are idempotent (safe to re-run)
**Database Persistence:**
- Job status stored in `scheduler.db` survives restarts
- Completed preps are never re-done
- Failed jobs stay marked as 'failed' (can be reset manually to 'pending' if needed)
## File Naming
Place your raw video files in `RAW_DIR` with names that contain the title from NocoDB.
Example:
- NocoDB Title: `"The Matrix"`
- Valid filenames: `The Matrix.mkv`, `the-matrix-1999.mp4`, `[1080p] The Matrix.avi`
The scheduler uses glob matching to find files containing the title.
## Streaming Watchdog
The scheduler includes an active watchdog that monitors streaming:
**Stream Monitoring (checked every 10 seconds):**
- ✅ Detects if ffmpeg stream crashes or exits unexpectedly
- ✅ Automatically restarts stream at the correct playback position
- ✅ Calculates elapsed time since stream start
- ✅ Seeks to correct position using `ffmpeg -ss` flag
- ✅ Limits restarts to 10 attempts to prevent infinite loops
- ✅ Marks stream as 'done' when video completes normally
- ✅ Marks stream as 'failed' after 10 failed restart attempts
**How It Works:**
1. Stream starts and status set to 'streaming' (not 'done')
2. Watchdog checks every 10 seconds if stream process is running
3. If crashed: calculates elapsed time and restarts with seek
4. If completed normally (exit code 0): marks as 'done'
5. If video duration exceeded: marks as 'done'
**Example:**
```
Stream starts at 21:00:00
Stream crashes at 21:05:30 (5.5 minutes in)
Watchdog detects crash within 10 seconds
Restarts stream with: ffmpeg -ss 330 -i video.mp4
Stream resumes from 5:30 mark
```
## Error Handling
The scheduler includes robust error handling:
- **Automatic Retries**: Failed operations are retried up to 3 times with exponential backoff
- **Database Logging**: All operations and errors are logged to the `log` column
- **Status Tracking**: Jobs are marked as 'failed' if all retries are exhausted
- **File Verification**: Checks that subtitle files and encoded videos were created successfully
## Monitoring
### Check Job Status
Query the SQLite database:
```bash
sqlite3 scheduler.db "SELECT nocodb_id, title, prep_status, play_status FROM jobs;"
```
### View Job Logs
```bash
sqlite3 scheduler.db "SELECT nocodb_id, title, log FROM jobs WHERE nocodb_id='YOUR_JOB_ID';"
```
### Console Output
The scheduler logs to stdout with timestamps:
- INFO: Normal operations
- WARNING: Retry attempts
- ERROR: Failures
## Troubleshooting
### Configuration Validation Failed
- Ensure all required environment variables are set
- Check that NOCODB_URL is accessible
- Verify NOCODB_TOKEN has proper permissions
### No Files Found Matching Title
- Check that video files exist in RAW_DIR
- Ensure filenames contain the NocoDB title (case-insensitive glob matching)
- Review the log column in the database for exact error messages
### Subtitle Generation Failed
- Verify whisper.cpp is installed: `which whisper.cpp`
- Check WHISPER_MODEL path is correct
- Ensure the model file exists and is readable
### Video Encoding Failed
- Confirm VAAPI is available: `ffmpeg -hwaccels | grep vaapi`
- Check VAAPI device exists: `ls -l /dev/dri/renderD128`
- Verify subtitle file was created successfully
- Review FFmpeg error output in the log column
### Streaming Failed
- Test RTMP server connectivity: `ffmpeg -re -i test.mp4 -c copy -f flv rtmp://your_server/live`
- Ensure final video file exists
- Check network connectivity to RTMP server
## Development
### Database Schema
```sql
CREATE TABLE jobs (
nocodb_id TEXT PRIMARY KEY,
title TEXT,
run_at TIMESTAMP,
prep_at TIMESTAMP,
raw_path TEXT,
final_path TEXT,
prep_status TEXT, -- pending/done/failed/skipped
play_status TEXT, -- pending/streaming/done/failed/skipped
log TEXT,
stream_start_time TIMESTAMP, -- When streaming started (for restart seek)
stream_retry_count INTEGER -- Number of stream restart attempts
);
```
**Status Values:**
- `play_status='streaming'`: Stream is actively running
- `play_status='done'`: Stream completed successfully
- `play_status='failed'`: Stream failed after 10 restart attempts
- `play_status='skipped'`: Too late to stream (>15 min past scheduled time)
### Customization
Use environment variables to customize:
- `NOCODB_SYNC_INTERVAL_SECONDS` - How often to check NocoDB for new jobs (default: 60)
- `WATCHDOG_CHECK_INTERVAL_SECONDS` - How often to monitor streams (default: 10)
- `STREAM_GRACE_PERIOD_MINUTES` - Late start tolerance (default: 15)
Edit `agent.py` to customize:
- Retry attempts and delay (agent.py:99, 203, 232)
- Preparation time offset (currently 6 hours, see sync function)
- Stream restart limit (agent.py:572)
- Subtitle styling (agent.py:246)
- Video encoding parameters (agent.py:248-250)
## License
This project is provided as-is without warranty.

486
TESTING.md Normal file
View File

@ -0,0 +1,486 @@
# Movie Scheduler - Testing Environment
Complete Docker-based testing environment for the movie scheduler. Fire and forget - just run one command and everything is set up automatically!
## Quick Start
```bash
./start-test-environment.sh
```
That's it! The script will:
1. Build all Docker containers
2. Start PostgreSQL, NocoDB, and RTMP server
3. Initialize NocoDB with test data
4. Download a test video (Big Buck Bunny)
5. Start the scheduler
## Services
After running the startup script, you'll have:
### NocoDB Dashboard
- **URL**: http://localhost:8080
- **Email**: admin@test.com
- **Password**: admin123
Login and view the "Movies" table to see scheduled movies and their status.
### RTMP Streaming Server
- **Stream URL**: rtmp://localhost:1935/live/stream
- **Statistics**: http://localhost:8081/stat
This is where processed movies will be streamed.
### Movie Scheduler
Runs in the background, automatically:
- Syncs movies from NocoDB every 60 seconds (configurable)
- Monitors stream health every 10 seconds (configurable)
- Prepares videos 6 hours before scheduled time
- Streams at the scheduled time
## Testing the Workflow
### 1. View Initial Data
The test environment comes with a pre-loaded movie:
- **Title**: BigBuckBunny
- **Scheduled**: 5 minutes from startup
Open NocoDB to see this entry.
### 2. Monitor the Scheduler
Watch the scheduler logs:
```bash
docker compose logs -f scheduler
```
You'll see:
- Syncing from NocoDB
- Preparation jobs (if within 6 hours of scheduled time)
- Subtitle generation
- Video encoding
- Streaming start
### 3. Check the Database
Query the local SQLite database:
```bash
sqlite3 scheduler.db "SELECT nocodb_id, title, prep_status, play_status FROM jobs;"
```
View detailed logs:
```bash
sqlite3 scheduler.db "SELECT title, log FROM jobs WHERE title='BigBuckBunny';"
```
### 4. Watch the Stream
When a movie starts streaming, watch it with VLC:
```bash
vlc rtmp://localhost:1935/live/stream
```
Or use ffplay:
```bash
ffplay rtmp://localhost:1935/live/stream
```
### 5. Add Your Own Movies
#### Option A: Through NocoDB UI
1. Open http://localhost:8080
2. Navigate to the Movies table
3. Click "+ Add New Row"
4. Fill in:
- **Title**: Name matching your video file
- **Date**: Schedule time (ISO format: 2024-01-21T20:00:00)
- Other fields are optional
5. Place video file in `raw_movies` volume
#### Option B: Add Video File Manually
Copy a video to the raw movies volume:
```bash
# Find the volume name
docker volume ls | grep raw_movies
# Copy your video
docker run --rm -v scheduler_raw_movies:/data -v $(pwd):/host alpine \
cp /host/your-movie.mp4 /data/
```
Then add the entry in NocoDB with matching Title.
## Testing Restart & Recovery
### Test Overdue Job Processing
1. Add a movie scheduled for 15 minutes from now
2. Stop the scheduler:
```bash
docker compose stop scheduler
```
3. Wait 20 minutes (past the prep time)
4. Restart the scheduler:
```bash
docker compose start scheduler
docker compose logs -f scheduler
```
5. Watch it immediately process the overdue prep job
### Test 15-Minute Grace Period for Streaming
1. Add a movie scheduled for 5 minutes from now (with prep already done)
2. Stop the scheduler after prep completes:
```bash
docker compose stop scheduler
```
3. Wait 10 minutes (past streaming time but within 15-min grace period)
4. Restart and watch logs:
```bash
docker compose start scheduler
docker compose logs -f scheduler
```
5. Stream should start immediately with "Starting stream X.X minutes late" message
### Test Skipping Late Streams (>15 minutes)
1. Add a movie scheduled for 5 minutes from now
2. Let prep complete, then stop the scheduler:
```bash
docker compose stop scheduler
```
3. Wait 20 minutes (past the 15-minute grace period)
4. Restart and watch logs:
```bash
docker compose start scheduler
docker compose logs -f scheduler
```
5. Job should be marked as 'skipped' with "more than 15 minutes late"
### Test Skipping Unprepared Expired Jobs
1. Add a movie scheduled for 5 minutes from now
2. Stop the scheduler before prep starts:
```bash
docker compose stop scheduler
```
3. Wait 10 minutes (past both prep and streaming time)
4. Restart and watch logs:
```bash
docker compose start scheduler
docker compose logs -f scheduler
```
5. Job should be marked as 'skipped' (too late to prep)
### Test Recovery from Crash During Processing
1. Start processing a large video file
2. Kill the scheduler during encoding:
```bash
docker compose kill scheduler
```
3. Restart:
```bash
docker compose start scheduler
```
4. Watch it retry the entire operation from the beginning
## Testing Stream Watchdog
### Test Stream Crash and Auto-Restart
1. Start a stream:
```bash
docker compose logs -f scheduler
```
2. Find the streaming ffmpeg process and kill it:
```bash
# In another terminal
docker compose exec scheduler ps aux | grep ffmpeg
docker compose exec scheduler kill -9 <PID>
```
3. Watch the scheduler logs - within 10 seconds it should:
- Detect the crash
- Calculate elapsed playback time
- Restart with seek to correct position
- Log: "Restarting stream at position X.Xs"
### Test Stream Restart with Correct Position
1. Start a test stream with a longer video
2. Let it run for 2-3 minutes
3. Kill the ffmpeg process
4. Watch logs confirm restart with seek:
```
Restarting stream for job XXX at position 180.0s (attempt 2)
```
5. Use VLC to confirm playback resumed at correct position
### Test Stream Completion Detection
1. Use a short test video (1-2 minutes)
2. Watch stream until completion
3. Check logs - should show:
```
Stream completed successfully for job XXX
```
4. Check database - play_status should be 'done'
### Test Retry Limit
1. Break the RTMP server or use invalid RTMP URL
2. Start a stream - it will crash immediately
3. Watch the scheduler attempt 10 restarts
4. After 10 attempts, should mark as 'failed':
```
Stream retry limit exceeded for job XXX
ERROR: Stream failed after 10 restart attempts
```
### Test Network Interruption Recovery
1. Start a stream
2. Stop the RTMP server:
```bash
docker compose stop rtmp
```
3. Watch stream fail and retry
4. Restart RTMP:
```bash
docker compose start rtmp
```
5. Stream should resume at correct position
## Testing Error Handling
### Test Retry Logic
1. Stop the RTMP server:
```bash
docker compose stop rtmp
```
2. Watch the scheduler handle the failure and retry:
```bash
docker compose logs -f scheduler
```
3. Restart RTMP:
```bash
docker compose start rtmp
```
### Test Missing File
1. Add a movie entry in NocoDB with a title that doesn't match any file
2. Set the Date to trigger immediately
3. Watch the scheduler log the error and mark it as failed
### Test Subtitle Generation Failure
1. Add an invalid video file (corrupted or wrong format)
2. Watch the scheduler attempt retries with exponential backoff
## Viewing Results
### Processed Videos
List final videos with burned-in subtitles:
```bash
docker run --rm -v scheduler_final_movies:/data alpine ls -lh /data
```
### Raw Videos
List source videos:
```bash
docker run --rm -v scheduler_raw_movies:/data alpine ls -lh /data
```
### RTMP Recordings (if enabled)
Recordings are saved to the rtmp_recordings volume:
```bash
docker run --rm -v scheduler_rtmp_recordings:/data alpine ls -lh /data/recordings
```
## Customization
### Change Test Video Scheduling
Edit `init-data.sh` before starting, around line 130:
```python
"Date": (datetime.now() + timedelta(minutes=5)).isoformat(),
```
Change `minutes=5` to adjust when the test movie is scheduled.
### Use Different NocoDB Credentials
Edit `docker-compose.yml`:
```yaml
environment:
NOCODB_EMAIL: "your@email.com"
NOCODB_PASSWORD: "yourpassword"
```
### Test with Real NocoDB Instance
Comment out the `nocodb` and `postgres` services in `docker-compose.yml` and update the `init` service environment:
```yaml
environment:
NOCODB_URL: "https://your-nocodb-instance.com"
NOCODB_EMAIL: "your@email.com"
NOCODB_PASSWORD: "yourpassword"
```
## Troubleshooting
### Services Won't Start
Check if ports are already in use:
```bash
lsof -i :8080 # NocoDB
lsof -i :1935 # RTMP
lsof -i :8081 # RTMP stats
```
### Initialization Fails
View init logs:
```bash
docker compose logs init
```
Common issues:
- NocoDB not ready yet (increase wait time in init-data.sh)
- Network connectivity issues
- Insufficient disk space
### Scheduler Not Processing
Check if it's running:
```bash
docker compose ps scheduler
```
View logs for errors:
```bash
docker compose logs scheduler
```
Restart if needed:
```bash
docker compose restart scheduler
```
### Video Download Fails
The init script downloads Big Buck Bunny from Google's test video bucket. If this fails:
1. Check internet connectivity
2. Manually download and copy to raw_movies volume
3. Or use your own test video
### VAAPI Not Available
The Docker container runs without GPU access by default (CPU encoding fallback). To enable VAAPI:
Edit `docker-compose.yml` scheduler service:
```yaml
scheduler:
devices:
- /dev/dri:/dev/dri
group_add:
- video
```
## Cleanup
### Stop Everything
```bash
docker compose down
```
### Remove All Data (including volumes)
```bash
docker compose down -v
```
This removes:
- All containers
- All volumes (database, videos, recordings)
- Network
### Keep Database, Remove Containers
```bash
docker compose down
# Volumes persist - next startup will reuse existing data
```
## Architecture
```
┌─────────────┐
│ PostgreSQL │
└──────┬──────┘
┌──────▼──────┐ ┌─────────────┐
│ NocoDB │◄─────┤ Init │ Downloads test video
└──────┬──────┘ │ Container │ Creates test data
│ └─────────────┘
┌──────▼──────┐
│ Scheduler │ Reads from NocoDB
└──────┬──────┘ Generates subtitles
│ Encodes videos
│ Manages streaming
┌──────▼──────┐
│ RTMP Server │ Streams processed videos
└─────────────┘
```
## Production Considerations
This test environment uses:
- Embedded PostgreSQL (fine for testing)
- Local volumes (fine for testing)
- Simple authentication (change for production)
- No SSL/TLS (add for production)
- CPU-only encoding (enable VAAPI for production)
For production deployment:
1. Use external PostgreSQL database
2. Configure proper authentication and secrets
3. Enable SSL/TLS for NocoDB
4. Mount persistent storage for videos
5. Enable GPU acceleration (VAAPI)
6. Set up proper monitoring and logging
7. Configure backup for NocoDB database
## Next Steps
Once you've tested the environment:
1. Review the logs to understand the workflow
2. Try adding your own movies
3. Test error scenarios
4. Adjust timing and configuration
5. Deploy to production with proper setup
## Support
For issues with:
- **Scheduler code**: Check `agent.py` and logs
- **Docker setup**: Check `docker-compose.yml` and service logs
- **NocoDB**: Visit https://docs.nocodb.com
- **RTMP streaming**: Check nginx-rtmp documentation
## License
This test environment is provided as-is for development and testing purposes.

684
agent.py Normal file
View File

@ -0,0 +1,684 @@
import requests
import sqlite3
import time
import subprocess
import hashlib
import os
import sys
import logging
from datetime import datetime, timedelta
from pathlib import Path
from functools import wraps
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
# Load configuration from environment variables
NOCODB_URL = os.getenv("NOCODB_URL")
NOCODB_TOKEN = os.getenv("NOCODB_TOKEN")
RTMP_SERVER = os.getenv("RTMP_SERVER")
RAW_DIR = Path(os.getenv("RAW_DIR", "/root/surowe_filmy"))
FINAL_DIR = Path(os.getenv("FINAL_DIR", "/root/przygotowane_filmy"))
WHISPER_MODEL = os.getenv("WHISPER_MODEL", "/root/models/ggml-base.bin")
VAAPI_DEVICE = os.getenv("VAAPI_DEVICE", "/dev/dri/renderD128")
STREAM_GRACE_PERIOD_MINUTES = int(os.getenv("STREAM_GRACE_PERIOD_MINUTES", "15"))
# Timing configuration
NOCODB_SYNC_INTERVAL_SECONDS = int(os.getenv("NOCODB_SYNC_INTERVAL_SECONDS", "60"))
WATCHDOG_CHECK_INTERVAL_SECONDS = int(os.getenv("WATCHDOG_CHECK_INTERVAL_SECONDS", "10"))
# Configuration validation
def validate_config():
"""Validate required environment variables on startup."""
required_vars = {
"NOCODB_URL": NOCODB_URL,
"NOCODB_TOKEN": NOCODB_TOKEN,
"RTMP_SERVER": RTMP_SERVER
}
missing = [name for name, value in required_vars.items() if not value]
if missing:
logger.error(f"Missing required environment variables: {', '.join(missing)}")
logger.error("Please set the following environment variables:")
logger.error(" NOCODB_URL - NocoDB API endpoint")
logger.error(" NOCODB_TOKEN - Authentication token")
logger.error(" RTMP_SERVER - Streaming destination")
sys.exit(1)
# Validate directories exist or can be created
for dir_path in [RAW_DIR, FINAL_DIR]:
try:
dir_path.mkdir(parents=True, exist_ok=True)
except Exception as e:
logger.error(f"Cannot create directory {dir_path}: {e}")
sys.exit(1)
logger.info("Configuration validated successfully")
logger.info(f"RAW_DIR: {RAW_DIR}")
logger.info(f"FINAL_DIR: {FINAL_DIR}")
logger.info(f"WHISPER_MODEL: {WHISPER_MODEL}")
logger.info(f"VAAPI_DEVICE: {VAAPI_DEVICE}")
logger.info(f"STREAM_GRACE_PERIOD: {STREAM_GRACE_PERIOD_MINUTES} minutes")
logger.info(f"NOCODB_SYNC_INTERVAL: {NOCODB_SYNC_INTERVAL_SECONDS} seconds")
logger.info(f"WATCHDOG_CHECK_INTERVAL: {WATCHDOG_CHECK_INTERVAL_SECONDS} seconds")
# Database setup
db = sqlite3.connect("scheduler.db", check_same_thread=False)
db.row_factory = sqlite3.Row
# Ensure table exists with log column and streaming metadata
db.execute("""
CREATE TABLE IF NOT EXISTS jobs (
nocodb_id TEXT PRIMARY KEY,
title TEXT,
run_at TIMESTAMP,
prep_at TIMESTAMP,
raw_path TEXT,
final_path TEXT,
prep_status TEXT,
play_status TEXT,
log TEXT,
stream_start_time TIMESTAMP,
stream_retry_count INTEGER DEFAULT 0
)
""")
db.commit()
# Add new columns if they don't exist (for existing databases)
try:
db.execute("ALTER TABLE jobs ADD COLUMN stream_start_time TIMESTAMP")
db.commit()
except:
pass
try:
db.execute("ALTER TABLE jobs ADD COLUMN stream_retry_count INTEGER DEFAULT 0")
db.commit()
except:
pass
# Track streaming process
streaming_process = None
current_streaming_job = None
def log_to_db(nocodb_id, message):
"""Append a timestamped log message to the database log column."""
timestamp = datetime.now().isoformat()
log_entry = f"[{timestamp}] {message}\n"
try:
db.execute("""
UPDATE jobs SET log = log || ? WHERE nocodb_id = ?
""", (log_entry, nocodb_id))
db.commit()
logger.info(f"Job {nocodb_id}: {message}")
except Exception as e:
logger.error(f"Failed to log to database: {e}")
def retry_with_backoff(max_attempts=3, base_delay=1):
"""Decorator to retry a function with exponential backoff."""
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
for attempt in range(1, max_attempts + 1):
try:
return func(*args, **kwargs)
except Exception as e:
if attempt == max_attempts:
logger.error(f"{func.__name__} failed after {max_attempts} attempts: {e}")
raise
delay = base_delay * (2 ** (attempt - 1))
logger.warning(f"{func.__name__} attempt {attempt} failed: {e}. Retrying in {delay}s...")
time.sleep(delay)
return wrapper
return decorator
@retry_with_backoff(max_attempts=3)
def sync():
"""Sync jobs from NocoDB."""
try:
response = requests.get(
NOCODB_URL,
headers={"xc-token": NOCODB_TOKEN},
timeout=30
)
response.raise_for_status()
rows = response.json().get("list", [])
logger.info(f"Fetched {len(rows)} jobs from NocoDB")
for r in rows:
try:
run_at = datetime.fromisoformat(r["Date"])
prep_at = run_at - timedelta(hours=6)
# Preserve existing status and streaming data on sync
existing = db.execute("SELECT * FROM jobs WHERE nocodb_id=?", (r["Id"],)).fetchone()
if existing:
db.execute("""
UPDATE jobs SET title=?, run_at=?, prep_at=?
WHERE nocodb_id=?
""", (r["Title"], run_at, prep_at, r["Id"]))
else:
db.execute("""
INSERT INTO jobs (nocodb_id, title, run_at, prep_at, raw_path, final_path,
prep_status, play_status, log, stream_start_time, stream_retry_count)
VALUES (?,?,?,?,?,?,?,?,?,?,?)
""", (r["Id"], r["Title"], run_at, prep_at, None, None, 'pending', 'pending', '', None, 0))
except Exception as e:
logger.error(f"Failed to process row {r.get('Id', 'unknown')}: {e}")
db.commit()
except requests.exceptions.RequestException as e:
logger.error(f"Failed to sync from NocoDB: {e}")
raise
def take_prep():
"""Get the next job ready for preparation."""
try:
# First, mark jobs as skipped if both prep and run times have passed
db.execute("""
UPDATE jobs
SET prep_status='skipped', play_status='skipped',
log = log || ?
WHERE prep_status='pending'
AND run_at <= CURRENT_TIMESTAMP
""", (f"[{datetime.now().isoformat()}] Job skipped - streaming time already passed\n",))
db.commit()
# Get next overdue or upcoming prep job
c = db.execute("""
SELECT * FROM jobs WHERE prep_status='pending' AND prep_at <= CURRENT_TIMESTAMP LIMIT 1
""")
job = c.fetchone()
if job:
# Check if this is an overdue job
prep_at = datetime.fromisoformat(job["prep_at"])
if prep_at < datetime.now() - timedelta(minutes=5):
logger.warning(f"Processing overdue prep job: {job['nocodb_id']} - {job['title']} (was due {prep_at})")
return job
except Exception as e:
logger.error(f"Failed to query prep jobs: {e}")
return None
def take_play():
"""Get the next job ready for streaming with configurable grace period."""
try:
# Calculate grace period cutoff (default 15 minutes ago)
grace_period_cutoff = datetime.now() - timedelta(minutes=STREAM_GRACE_PERIOD_MINUTES)
# Mark jobs more than STREAM_GRACE_PERIOD_MINUTES overdue as skipped
db.execute("""
UPDATE jobs
SET play_status='skipped',
log = log || ?
WHERE prep_status='done'
AND play_status='pending'
AND run_at < ?
""", (
f"[{datetime.now().isoformat()}] Streaming skipped - more than {STREAM_GRACE_PERIOD_MINUTES} minutes late\n",
grace_period_cutoff.isoformat()
))
db.commit()
# Get jobs ready to stream (on time or within 15-minute grace period)
c = db.execute("""
SELECT * FROM jobs
WHERE prep_status='done'
AND play_status='pending'
AND run_at <= CURRENT_TIMESTAMP
LIMIT 1
""")
job = c.fetchone()
if job:
# Check if this is a late start
run_at = datetime.fromisoformat(job["run_at"])
delay = datetime.now() - run_at
if delay > timedelta(seconds=30):
minutes_late = delay.total_seconds() / 60
logger.warning(f"Starting stream late: {job['nocodb_id']} - {job['title']} ({minutes_late:.1f} minutes after scheduled time)")
return job
except Exception as e:
logger.error(f"Failed to query play jobs: {e}")
return None
def run_subprocess(cmd, job_id, description, timeout=None):
"""Run a subprocess with error handling and logging."""
log_to_db(job_id, f"Starting: {description}")
logger.info(f"Running command: {' '.join(str(c) for c in cmd)}")
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=timeout,
check=True
)
if result.stdout:
logger.debug(f"stdout: {result.stdout[:500]}")
log_to_db(job_id, f"Completed: {description}")
return result
except subprocess.TimeoutExpired:
error_msg = f"Timeout: {description} exceeded {timeout}s"
log_to_db(job_id, f"ERROR: {error_msg}")
raise Exception(error_msg)
except subprocess.CalledProcessError as e:
error_msg = f"Command failed with code {e.returncode}: {e.stderr[:500] if e.stderr else 'no error output'}"
log_to_db(job_id, f"ERROR: {description} - {error_msg}")
raise Exception(error_msg)
@retry_with_backoff(max_attempts=3, base_delay=2)
def generate_subtitles(raw_file, job_id):
"""Generate subtitles using whisper.cpp."""
srt_file = raw_file.with_suffix(".srt")
# Remove existing subtitle file if present
if srt_file.exists():
srt_file.unlink()
log_to_db(job_id, f"Removed existing subtitle file: {srt_file}")
# Run whisper.cpp with correct format: whisper.cpp -m <model> -f <file> -osrt
cmd = [
"whisper.cpp",
"-m", str(WHISPER_MODEL),
"-f", str(raw_file),
"-osrt"
]
run_subprocess(cmd, job_id, "Subtitle generation with whisper.cpp", timeout=3600)
# Verify subtitle file was created
if not srt_file.exists():
error_msg = f"Subtitle file not created: {srt_file}"
log_to_db(job_id, f"ERROR: {error_msg}")
raise Exception(error_msg)
log_to_db(job_id, f"Subtitle file created successfully: {srt_file}")
return srt_file
@retry_with_backoff(max_attempts=3, base_delay=2)
def encode_video_with_subtitles(raw_file, srt_file, final_file, job_id):
"""Encode video with burned-in subtitles using VAAPI."""
# Remove existing output file if present
if final_file.exists():
final_file.unlink()
log_to_db(job_id, f"Removed existing output file: {final_file}")
# FFmpeg command with VAAPI encoding (h264_vaapi instead of h264_qsv)
cmd = [
"ffmpeg",
"-hwaccel", "vaapi",
"-vaapi_device", VAAPI_DEVICE,
"-i", str(raw_file),
"-vf", f"subtitles={srt_file}:force_style=Fontname=Consolas,BackColour=&H80000000,Spacing=0.2,Outline=0,Shadow=0.75,format=yuv420p",
"-c:v", "h264_vaapi", # Changed from h264_qsv to h264_vaapi
"-qp", "23", # Quality parameter for VAAPI (similar to CRF)
"-c:a", "aac",
"-b:a", "192k",
"-movflags", "faststart",
"-y", # Overwrite output file
str(final_file)
]
run_subprocess(cmd, job_id, "Video encoding with VAAPI", timeout=7200)
# Verify output file was created
if not final_file.exists():
error_msg = f"Output video file not created: {final_file}"
log_to_db(job_id, f"ERROR: {error_msg}")
raise Exception(error_msg)
file_size = final_file.stat().st_size / (1024 * 1024) # Size in MB
log_to_db(job_id, f"Video encoded successfully: {final_file} ({file_size:.2f} MB)")
return final_file
def prepare_job(job):
"""Prepare a job: generate subtitles and encode video."""
job_id = job["nocodb_id"]
title = job["title"]
try:
log_to_db(job_id, f"Starting preparation for: {title}")
# Find raw video file
matching_files = list(RAW_DIR.glob(f"*{title}*"))
if not matching_files:
error_msg = f"No files found matching title: {title}"
log_to_db(job_id, f"ERROR: {error_msg}")
db.execute("UPDATE jobs SET prep_status='failed' WHERE nocodb_id=?", (job_id,))
db.commit()
return
raw_file = max(matching_files, key=lambda x: x.stat().st_mtime)
log_to_db(job_id, f"Found raw file: {raw_file}")
# Generate subtitles
srt_file = generate_subtitles(raw_file, job_id)
# Prepare output filename
final_file = FINAL_DIR / (raw_file.stem + ".converted.mp4")
# Encode video with subtitles
encode_video_with_subtitles(raw_file, srt_file, final_file, job_id)
# Update database
db.execute("""
UPDATE jobs SET prep_status='done', raw_path=?, final_path=? WHERE nocodb_id=?
""", (str(raw_file), str(final_file), job_id))
db.commit()
log_to_db(job_id, "Preparation completed successfully")
except Exception as e:
error_msg = f"Preparation failed: {e}"
log_to_db(job_id, f"ERROR: {error_msg}")
logger.error(f"Job {job_id} preparation failed: {e}")
db.execute("UPDATE jobs SET prep_status='failed' WHERE nocodb_id=?", (job_id,))
db.commit()
def get_video_duration(video_path):
"""Get video duration in seconds using ffprobe."""
try:
result = subprocess.run(
[
"ffprobe",
"-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
str(video_path)
],
capture_output=True,
text=True,
timeout=10
)
if result.returncode == 0 and result.stdout.strip():
return float(result.stdout.strip())
except Exception as e:
logger.error(f"Failed to get video duration: {e}")
return None
def stream_job(job, seek_seconds=0):
"""Start streaming a prepared job with optional seek position."""
global streaming_process, current_streaming_job
job_id = job["nocodb_id"]
final_path = job["final_path"]
try:
# Check if starting late or restarting
run_at = datetime.fromisoformat(job["run_at"])
delay = datetime.now() - run_at
if seek_seconds > 0:
log_to_db(job_id, f"Restarting stream at position {seek_seconds:.1f}s")
elif delay > timedelta(seconds=30):
minutes_late = delay.total_seconds() / 60
log_to_db(job_id, f"Starting stream {minutes_late:.1f} minutes late")
log_to_db(job_id, f"Starting stream to: {RTMP_SERVER}")
# Verify file exists
if not Path(final_path).exists():
error_msg = f"Final video file not found: {final_path}"
log_to_db(job_id, f"ERROR: {error_msg}")
db.execute("UPDATE jobs SET play_status='failed' WHERE nocodb_id=?", (job_id,))
db.commit()
return False
# Get video duration to know when it should finish
video_duration = get_video_duration(final_path)
if not video_duration:
logger.warning(f"Could not determine video duration for {final_path}")
# Stop previous stream if running
if streaming_process and streaming_process.poll() is None:
logger.info(f"Stopping previous stream (PID: {streaming_process.pid})")
streaming_process.terminate()
try:
streaming_process.wait(timeout=5)
except subprocess.TimeoutExpired:
streaming_process.kill()
log_to_db(job_id, "Stopped previous stream")
# Build ffmpeg command with optional seek
cmd = ["ffmpeg"]
if seek_seconds > 0:
# Seek to position (input seeking is faster)
cmd.extend(["-ss", str(seek_seconds)])
cmd.extend([
"-re", # Read input at native frame rate
"-i", final_path,
"-c", "copy", # Copy streams without re-encoding
"-f", "flv",
RTMP_SERVER
])
logger.info(f"Starting stream: {' '.join(cmd)}")
streaming_process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
# Record when this stream started (for calculating seek position on restart)
stream_start_time = datetime.now()
current_streaming_job = job_id
log_to_db(job_id, f"Stream started (PID: {streaming_process.pid})")
# Update database - set status to 'streaming', not 'done'
db.execute("""
UPDATE jobs
SET play_status='streaming',
stream_start_time=?,
stream_retry_count=stream_retry_count+1
WHERE nocodb_id=?
""", (stream_start_time.isoformat(), job_id))
db.commit()
return True
except Exception as e:
error_msg = f"Streaming failed: {e}"
log_to_db(job_id, f"ERROR: {error_msg}")
logger.error(f"Job {job_id} streaming failed: {e}")
db.execute("UPDATE jobs SET play_status='failed' WHERE nocodb_id=?", (job_id,))
db.commit()
return False
def monitor_and_restart_stream():
"""Monitor active stream and restart if it fails."""
global streaming_process, current_streaming_job
# Check if there's supposed to be an active stream
try:
active_stream = db.execute("""
SELECT * FROM jobs WHERE play_status='streaming' LIMIT 1
""").fetchone()
if not active_stream:
# No active stream expected
streaming_process = None
current_streaming_job = None
return
job_id = active_stream["nocodb_id"]
final_path = active_stream["final_path"]
stream_start_time = active_stream["stream_start_time"]
retry_count = active_stream["stream_retry_count"] or 0
# Check if process is still running
if streaming_process and streaming_process.poll() is None:
# Stream is running fine
return
# Stream is not running but should be
if streaming_process:
# Process exited - check if it was normal completion or error
return_code = streaming_process.returncode
if return_code == 0:
# Normal completion - video finished
logger.info(f"Stream completed successfully for job {job_id}")
log_to_db(job_id, "Stream completed successfully")
db.execute("UPDATE jobs SET play_status='done' WHERE nocodb_id=?", (job_id,))
db.commit()
streaming_process = None
current_streaming_job = None
return
else:
# Error exit
stderr = streaming_process.stderr.read().decode('utf-8')[-500:] if streaming_process.stderr else ""
logger.error(f"Stream crashed for job {job_id} with code {return_code}: {stderr}")
log_to_db(job_id, f"Stream crashed (exit code {return_code})")
# Calculate how much time has elapsed since stream started
if not stream_start_time:
logger.error(f"No stream start time recorded for job {job_id}")
db.execute("UPDATE jobs SET play_status='failed' WHERE nocodb_id=?", (job_id,))
db.commit()
return
start_time = datetime.fromisoformat(stream_start_time)
elapsed_seconds = (datetime.now() - start_time).total_seconds()
# Get video duration to check if we should still be streaming
video_duration = get_video_duration(final_path)
if video_duration and elapsed_seconds >= video_duration:
# Video should have finished by now
logger.info(f"Stream duration exceeded for job {job_id} - marking as done")
log_to_db(job_id, "Stream duration completed")
db.execute("UPDATE jobs SET play_status='done' WHERE nocodb_id=?", (job_id,))
db.commit()
streaming_process = None
current_streaming_job = None
return
# Check retry limit (max 10 restarts)
if retry_count >= 10:
logger.error(f"Stream retry limit exceeded for job {job_id}")
log_to_db(job_id, "ERROR: Stream failed after 10 restart attempts")
db.execute("UPDATE jobs SET play_status='failed' WHERE nocodb_id=?", (job_id,))
db.commit()
streaming_process = None
current_streaming_job = None
return
# Restart stream at correct position
logger.warning(f"Restarting stream for job {job_id} at position {elapsed_seconds:.1f}s (attempt {retry_count + 1})")
stream_job(active_stream, seek_seconds=elapsed_seconds)
except Exception as e:
logger.error(f"Error in stream monitor: {e}")
def main():
"""Main scheduler loop."""
logger.info("Starting scheduler...")
validate_config()
# Check for overdue jobs on startup
try:
grace_period_cutoff = datetime.now() - timedelta(minutes=STREAM_GRACE_PERIOD_MINUTES)
overdue_prep = db.execute("""
SELECT COUNT(*) as count FROM jobs
WHERE prep_status='pending' AND prep_at <= CURRENT_TIMESTAMP AND run_at > CURRENT_TIMESTAMP
""").fetchone()
skipped_prep = db.execute("""
SELECT COUNT(*) as count FROM jobs
WHERE prep_status='pending' AND run_at <= CURRENT_TIMESTAMP
""").fetchone()
overdue_stream = db.execute("""
SELECT COUNT(*) as count FROM jobs
WHERE prep_status='done' AND play_status='pending' AND run_at <= CURRENT_TIMESTAMP AND run_at >= ?
""", (grace_period_cutoff.isoformat(),)).fetchone()
skipped_stream = db.execute("""
SELECT COUNT(*) as count FROM jobs
WHERE prep_status='done' AND play_status='pending' AND run_at < ?
""", (grace_period_cutoff.isoformat(),)).fetchone()
if overdue_prep and overdue_prep["count"] > 0:
logger.warning(f"Found {overdue_prep['count']} overdue prep job(s) - will process immediately")
if skipped_prep and skipped_prep["count"] > 0:
logger.warning(f"Found {skipped_prep['count']} unprepared job(s) past streaming time - will be marked as skipped")
if overdue_stream and overdue_stream["count"] > 0:
logger.warning(f"Found {overdue_stream['count']} overdue streaming job(s) - will start within grace period ({STREAM_GRACE_PERIOD_MINUTES}min)")
if skipped_stream and skipped_stream["count"] > 0:
logger.warning(f"Found {skipped_stream['count']} streaming job(s) more than {STREAM_GRACE_PERIOD_MINUTES}min late - will be marked as skipped")
except Exception as e:
logger.error(f"Failed to check for overdue jobs: {e}")
logger.info("Scheduler is running. Press Ctrl+C to stop.")
try:
# Calculate how many iterations between syncs
sync_iterations = max(1, NOCODB_SYNC_INTERVAL_SECONDS // WATCHDOG_CHECK_INTERVAL_SECONDS)
logger.info(f"Main loop: checking every {WATCHDOG_CHECK_INTERVAL_SECONDS}s, syncing every {sync_iterations} iterations ({NOCODB_SYNC_INTERVAL_SECONDS}s)")
iteration = 0
while True:
try:
# Monitor and restart stream if needed (check every iteration)
monitor_and_restart_stream()
# Sync jobs from NocoDB at configured interval
if iteration % sync_iterations == 0:
sync()
# Process preparation jobs
job = take_prep()
if job:
logger.info(f"Processing prep job: {job['nocodb_id']} - {job['title']}")
prepare_job(job)
# Process streaming jobs
job = take_play()
if job:
logger.info(f"Processing play job: {job['nocodb_id']} - {job['title']}")
stream_job(job)
except Exception as e:
logger.error(f"Error in main loop: {e}")
# Sleep between iterations
time.sleep(WATCHDOG_CHECK_INTERVAL_SECONDS)
iteration += 1
except KeyboardInterrupt:
logger.info("Scheduler stopped by user")
if streaming_process and streaming_process.poll() is None:
logger.info("Stopping active stream...")
streaming_process.terminate()
finally:
db.close()
if __name__ == "__main__":
main()

74
docker-compose.prod.yml Normal file
View File

@ -0,0 +1,74 @@
version: '3.8'
services:
scheduler:
build:
context: .
dockerfile: Dockerfile
container_name: movie_scheduler
restart: unless-stopped
# Environment variables - USE SECRETS IN PRODUCTION
env_file:
- .env.production
# GPU access for VAAPI encoding
devices:
- /dev/dri:/dev/dri
group_add:
- video
- render
volumes:
# Persistent database
- ./scheduler.db:/app/scheduler.db
# Video storage - mount your actual storage paths
- /mnt/storage/raw_movies:/raw_movies:ro
- /mnt/storage/final_movies:/final_movies
# Whisper model (download once, reuse)
- /opt/models:/models:ro
# Logs (optional - use if not using external logging)
- ./logs:/app/logs
# Logging configuration
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "5"
# Resource limits
deploy:
resources:
limits:
cpus: '4.0'
memory: 4G
reservations:
cpus: '1.0'
memory: 1G
# Health check
healthcheck:
test: ["CMD-SHELL", "pgrep -f agent.py || exit 1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
# Network
networks:
- scheduler_network
# Security
security_opt:
- no-new-privileges:true
read_only: false
tmpfs:
- /tmp
networks:
scheduler_network:
driver: bridge

112
docker-compose.yml Normal file
View File

@ -0,0 +1,112 @@
version: '3.8'
services:
# PostgreSQL database for NocoDB
postgres:
image: postgres:15-alpine
environment:
POSTGRES_DB: nocodb
POSTGRES_USER: nocodb
POSTGRES_PASSWORD: nocodb_password
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U nocodb"]
interval: 5s
timeout: 5s
retries: 5
networks:
- scheduler_network
# NocoDB service
nocodb:
image: nocodb/nocodb:latest
ports:
- "8080:8080"
environment:
NC_DB: "pg://postgres:5432?u=nocodb&p=nocodb_password&d=nocodb"
NC_AUTH_JWT_SECRET: "test_jwt_secret_key_12345"
NC_PUBLIC_URL: "http://localhost:8080"
depends_on:
postgres:
condition: service_healthy
volumes:
- nocodb_data:/usr/app/data
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/health || exit 1"]
interval: 10s
timeout: 5s
retries: 10
networks:
- scheduler_network
# RTMP Server for streaming
rtmp:
image: tiangolo/nginx-rtmp
ports:
- "1935:1935"
- "8081:80"
volumes:
- ./nginx-rtmp.conf:/etc/nginx/nginx.conf:ro
- rtmp_recordings:/tmp/recordings
networks:
- scheduler_network
# Initialization service - sets up test data
init:
build:
context: .
dockerfile: Dockerfile.init
depends_on:
nocodb:
condition: service_healthy
environment:
NOCODB_URL: "http://nocodb:8080"
NOCODB_EMAIL: "admin@test.com"
NOCODB_PASSWORD: "admin123"
volumes:
- raw_movies:/raw_movies
- config_data:/tmp
- ./init-data.sh:/init-data.sh:ro
networks:
- scheduler_network
command: ["/bin/bash", "/init-data.sh"]
# Movie scheduler service
scheduler:
build:
context: .
dockerfile: Dockerfile
depends_on:
nocodb:
condition: service_healthy
rtmp:
condition: service_started
init:
condition: service_completed_successfully
environment:
RAW_DIR: "/raw_movies"
FINAL_DIR: "/final_movies"
WHISPER_MODEL: "/models/ggml-base.bin"
volumes:
- raw_movies:/raw_movies
- final_movies:/final_movies
- config_data:/config:ro
- ./scheduler.db:/app/scheduler.db
- ./start-scheduler.sh:/start-scheduler.sh:ro
networks:
- scheduler_network
restart: unless-stopped
command: ["/bin/bash", "/start-scheduler.sh"]
volumes:
postgres_data:
nocodb_data:
raw_movies:
final_movies:
rtmp_recordings:
config_data:
networks:
scheduler_network:
driver: bridge

113
docker-helper.sh Executable file
View File

@ -0,0 +1,113 @@
#!/bin/bash
# Docker Compose command detection
if docker compose version &> /dev/null; then
COMPOSE="docker compose"
elif docker-compose --version &> /dev/null; then
COMPOSE="docker-compose"
else
echo "Error: Docker Compose not found!"
exit 1
fi
case "$1" in
start)
echo "Starting all services..."
$COMPOSE up -d
;;
stop)
echo "Stopping all services..."
$COMPOSE down
;;
restart)
echo "Restarting all services..."
$COMPOSE restart
;;
logs)
if [ -z "$2" ]; then
$COMPOSE logs -f
else
$COMPOSE logs -f "$2"
fi
;;
status)
$COMPOSE ps
;;
clean)
echo "Stopping and removing all containers and volumes..."
read -p "This will delete all data. Continue? (y/N) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
$COMPOSE down -v
echo "Cleanup complete!"
else
echo "Cancelled."
fi
;;
db)
echo "Opening scheduler database..."
if [ -f "scheduler.db" ]; then
sqlite3 scheduler.db
else
echo "Database file not found. Is the scheduler running?"
fi
;;
jobs)
echo "Current jobs:"
if [ -f "scheduler.db" ]; then
sqlite3 scheduler.db "SELECT nocodb_id, title, prep_status, play_status, datetime(run_at) as scheduled FROM jobs ORDER BY run_at;"
else
echo "Database file not found."
fi
;;
add-video)
if [ -z "$2" ]; then
echo "Usage: $0 add-video <path-to-video-file>"
exit 1
fi
if [ ! -f "$2" ]; then
echo "File not found: $2"
exit 1
fi
echo "Adding video to raw_movies volume..."
docker run --rm -v scheduler_raw_movies:/data -v "$(pwd):/host" alpine \
cp "/host/$2" /data/
echo "✓ Video added: $(basename "$2")"
echo "Now add an entry in NocoDB with Title matching: $(basename "$2" | cut -d. -f1)"
;;
stream)
echo "To watch the stream, use one of these commands:"
echo ""
echo "VLC:"
echo " vlc rtmp://localhost:1935/live/stream"
echo ""
echo "ffplay:"
echo " ffplay rtmp://localhost:1935/live/stream"
echo ""
echo "MPV:"
echo " mpv rtmp://localhost:1935/live/stream"
;;
*)
echo "Movie Scheduler - Docker Helper"
echo ""
echo "Usage: $0 <command> [arguments]"
echo ""
echo "Commands:"
echo " start Start all services"
echo " stop Stop all services"
echo " restart Restart all services"
echo " logs [service] View logs (all or specific service)"
echo " status Show service status"
echo " clean Stop and remove all data"
echo " db Open scheduler database (SQLite)"
echo " jobs List all jobs and their status"
echo " add-video <file> Add video to raw_movies volume"
echo " stream Show commands to watch the stream"
echo ""
echo "Examples:"
echo " $0 start"
echo " $0 logs scheduler"
echo " $0 add-video my-movie.mp4"
echo " $0 jobs"
;;
esac

261
init-data.sh Executable file
View File

@ -0,0 +1,261 @@
#!/bin/bash
set -e
echo "========================================"
echo "NocoDB & Test Data Initialization"
echo "========================================"
NOCODB_URL=${NOCODB_URL:-"http://nocodb:8080"}
NOCODB_EMAIL=${NOCODB_EMAIL:-"admin@test.com"}
NOCODB_PASSWORD=${NOCODB_PASSWORD:-"admin123"}
# Wait for NocoDB to be fully ready
echo "Waiting for NocoDB to be ready..."
for i in {1..30}; do
if curl -sf "${NOCODB_URL}/api/v1/health" > /dev/null 2>&1; then
echo "NocoDB is ready!"
break
fi
echo "Attempt $i/30: NocoDB not ready yet, waiting..."
sleep 2
done
# Download test video (Big Buck Bunny - open source test video)
echo ""
echo "Downloading test video..."
if [ ! -f "/raw_movies/BigBuckBunny.mp4" ]; then
wget -q -O /raw_movies/BigBuckBunny.mp4 \
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4" \
|| echo "Failed to download test video, but continuing..."
if [ -f "/raw_movies/BigBuckBunny.mp4" ]; then
echo "✓ Test video downloaded: BigBuckBunny.mp4"
else
echo "⚠ Could not download test video - you'll need to provide one manually"
fi
else
echo "✓ Test video already exists"
fi
# Run Python script to setup NocoDB
echo ""
echo "Setting up NocoDB database and table..."
python3 << 'PYTHON_SCRIPT'
import requests
import json
import sys
import os
from datetime import datetime, timedelta
NOCODB_URL = os.getenv("NOCODB_URL", "http://nocodb:8080")
EMAIL = os.getenv("NOCODB_EMAIL", "admin@test.com")
PASSWORD = os.getenv("NOCODB_PASSWORD", "admin123")
print(f"Connecting to NocoDB at {NOCODB_URL}")
# Step 1: Sign up (first time) or sign in
try:
# Try to sign up first
response = requests.post(
f"{NOCODB_URL}/api/v1/auth/user/signup",
json={
"email": EMAIL,
"password": PASSWORD
},
timeout=10
)
if response.status_code == 200:
print("✓ New user created successfully")
auth_data = response.json()
else:
# If signup fails, try signin
response = requests.post(
f"{NOCODB_URL}/api/v1/auth/user/signin",
json={
"email": EMAIL,
"password": PASSWORD
},
timeout=10
)
response.raise_for_status()
print("✓ Signed in successfully")
auth_data = response.json()
except Exception as e:
print(f"✗ Authentication failed: {e}")
sys.exit(1)
token = auth_data.get("token")
if not token:
print("✗ No token received from NocoDB")
sys.exit(1)
print(f"✓ Got authentication token")
headers = {
"xc-auth": token,
"Content-Type": "application/json"
}
# Step 2: Create a new base (project)
try:
response = requests.post(
f"{NOCODB_URL}/api/v1/db/meta/projects/",
headers=headers,
json={
"title": "MovieScheduler",
"type": "database"
},
timeout=10
)
if response.status_code in [200, 201]:
base = response.json()
print(f"✓ Created base: {base.get('title')}")
else:
# Base might already exist, try to get it
response = requests.get(
f"{NOCODB_URL}/api/v1/db/meta/projects/",
headers=headers,
timeout=10
)
bases = response.json().get("list", [])
base = next((b for b in bases if b.get("title") == "MovieScheduler"), bases[0] if bases else None)
if not base:
print("✗ Could not create or find base")
sys.exit(1)
print(f"✓ Using existing base: {base.get('title')}")
except Exception as e:
print(f"✗ Base creation failed: {e}")
sys.exit(1)
base_id = base.get("id")
print(f"Base ID: {base_id}")
# Step 3: Get or create table
try:
# List existing tables
response = requests.get(
f"{NOCODB_URL}/api/v1/db/meta/projects/{base_id}/tables",
headers=headers,
timeout=10
)
tables = response.json().get("list", [])
# Check if Movies table exists
movies_table = next((t for t in tables if t.get("title") == "Movies"), None)
if not movies_table:
# Create Movies table
response = requests.post(
f"{NOCODB_URL}/api/v1/db/meta/projects/{base_id}/tables",
headers=headers,
json={
"table_name": "Movies",
"title": "Movies",
"columns": [
{"column_name": "Id", "title": "Id", "uidt": "ID"},
{"column_name": "Title", "title": "Title", "uidt": "SingleLineText"},
{"column_name": "Language", "title": "Language", "uidt": "SingleLineText"},
{"column_name": "Year", "title": "Year", "uidt": "Year"},
{"column_name": "Date", "title": "Date", "uidt": "DateTime"},
{"column_name": "Image", "title": "Image", "uidt": "URL"},
{"column_name": "IMDB", "title": "IMDB", "uidt": "URL"},
{"column_name": "Time", "title": "Time", "uidt": "Duration"},
{"column_name": "Kanal", "title": "Kanal", "uidt": "SingleLineText"},
{"column_name": "Obejrzane", "title": "Obejrzane", "uidt": "Checkbox"}
]
},
timeout=10
)
response.raise_for_status()
movies_table = response.json()
print(f"✓ Created table: Movies")
else:
print(f"✓ Using existing table: Movies")
except Exception as e:
print(f"✗ Table creation failed: {e}")
sys.exit(1)
table_id = movies_table.get("id")
print(f"Table ID: {table_id}")
# Step 4: Add test movie entries
test_movies = [
{
"Title": "BigBuckBunny",
"Language": "po angielsku; z napisami",
"Year": 2008,
"Date": (datetime.now() + timedelta(minutes=5)).isoformat(),
"Image": "https://peach.blender.org/wp-content/uploads/title_anouncement.jpg",
"IMDB": "https://www.imdb.com/title/tt1254207/",
"Time": "9:56",
"Kanal": "TestTV",
"Obejrzane": False
},
{
"Title": "TestMovie",
"Language": "po angielsku; z napisami",
"Year": 2024,
"Date": (datetime.now() + timedelta(hours=1)).isoformat(),
"Image": "https://example.com/test.jpg",
"IMDB": "https://www.imdb.com/",
"Time": "1:30:00",
"Kanal": "TestTV",
"Obejrzane": False
}
]
try:
for movie in test_movies:
response = requests.post(
f"{NOCODB_URL}/api/v2/tables/{table_id}/records",
headers={"xc-token": token, "Content-Type": "application/json"},
json=movie,
timeout=10
)
if response.status_code in [200, 201]:
print(f"✓ Added movie: {movie['Title']}")
else:
print(f"⚠ Could not add movie {movie['Title']}: {response.text}")
except Exception as e:
print(f"⚠ Some movies could not be added: {e}")
# Step 5: Save configuration for scheduler
print("\n" + "="*50)
print("CONFIGURATION FOR SCHEDULER")
print("="*50)
print(f"NOCODB_URL: http://nocodb:8080/api/v2/tables/{table_id}/records")
print(f"NOCODB_TOKEN: {token}")
print(f"RTMP_SERVER: rtmp://rtmp:1935/live/stream")
print("="*50)
# Save to file that docker-compose can read
with open("/tmp/nocodb_config.env", "w") as f:
f.write(f"NOCODB_TABLE_ID={table_id}\n")
f.write(f"NOCODB_TOKEN={token}\n")
print("\n✓ Configuration saved to /tmp/nocodb_config.env")
print("\n✓ Initialization complete!")
PYTHON_SCRIPT
echo ""
echo "========================================"
echo "✓ All initialization steps completed!"
echo "========================================"
echo ""
echo "NocoDB Dashboard: http://localhost:8080"
echo " Email: ${NOCODB_EMAIL}"
echo " Password: ${NOCODB_PASSWORD}"
echo ""
echo "RTMP Server: rtmp://localhost:1935/live/stream"
echo "RTMP Stats: http://localhost:8081/stat"
echo ""
echo "The scheduler will start automatically."
echo "========================================"

66
nginx-rtmp.conf Normal file
View File

@ -0,0 +1,66 @@
worker_processes auto;
events {
worker_connections 1024;
}
# RTMP configuration
rtmp {
server {
listen 1935;
chunk_size 4096;
allow publish all;
application live {
live on;
record off;
# Allow playback from anywhere
allow play all;
# HLS settings (optional, for web playback)
hls on;
hls_path /tmp/hls;
hls_fragment 3;
hls_playlist_length 60;
# Record streams (optional)
record all;
record_path /tmp/recordings;
record_suffix -%Y%m%d-%H%M%S.flv;
}
}
}
# HTTP server for HLS playback
http {
server {
listen 80;
# HLS streaming endpoint
location /hls {
types {
application/vnd.apple.mpegurl m3u8;
video/mp2t ts;
}
root /tmp;
add_header Cache-Control no-cache;
add_header Access-Control-Allow-Origin *;
}
# RTMP statistics
location /stat {
rtmp_stat all;
rtmp_stat_stylesheet stat.xsl;
}
location /stat.xsl {
root /usr/local/nginx/html;
}
# Status page
location / {
root /usr/local/nginx/html;
index index.html;
}
}
}

1
requirements.txt Normal file
View File

@ -0,0 +1 @@
requests>=2.25.0

BIN
scheduler.db Normal file

Binary file not shown.

56
scheduler.service Normal file
View File

@ -0,0 +1,56 @@
[Unit]
Description=Movie Scheduler - Automated video processing and streaming
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=scheduler
Group=scheduler
WorkingDirectory=/opt/scheduler
# Environment variables (or use EnvironmentFile)
Environment="NOCODB_URL=https://your-nocodb.com/api/v2/tables/YOUR_TABLE_ID/records"
Environment="NOCODB_TOKEN=YOUR_TOKEN_HERE"
Environment="RTMP_SERVER=rtmp://your-rtmp-server.com/live/stream"
Environment="RAW_DIR=/mnt/storage/raw_movies"
Environment="FINAL_DIR=/mnt/storage/final_movies"
Environment="WHISPER_MODEL=/opt/models/ggml-base.bin"
Environment="VAAPI_DEVICE=/dev/dri/renderD128"
Environment="STREAM_GRACE_PERIOD_MINUTES=15"
Environment="NOCODB_SYNC_INTERVAL_SECONDS=60"
Environment="WATCHDOG_CHECK_INTERVAL_SECONDS=10"
# Or use external environment file (recommended for secrets)
# EnvironmentFile=/etc/scheduler/scheduler.env
# Python virtual environment
ExecStart=/opt/scheduler/venv/bin/python /opt/scheduler/agent.py
# Restart policy
Restart=always
RestartSec=10
# Security hardening
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/mnt/storage /opt/scheduler
# Allow access to GPU for VAAPI
DeviceAllow=/dev/dri/renderD128 rw
SupplementaryGroups=video render
# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=scheduler
# Resource limits (adjust based on your needs)
LimitNOFILE=65536
MemoryMax=4G
CPUQuota=200%
[Install]
WantedBy=multi-user.target

92
scripts/backup.sh Executable file
View File

@ -0,0 +1,92 @@
#!/bin/bash
#
# Scheduler Backup Script
# Backs up database and configuration with rotation
#
set -e
# Configuration
BACKUP_DIR="${BACKUP_DIR:-/backup/scheduler}"
DB_PATH="${DB_PATH:-/opt/scheduler/scheduler.db}"
CONFIG_PATH="${CONFIG_PATH:-/etc/scheduler/scheduler.env}"
RETENTION_DAYS=30
COMPRESS_AFTER_DAYS=7
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
DATE=$(date +%Y%m%d_%H%M%S)
echo -e "${GREEN}=== Scheduler Backup - $(date) ===${NC}"
# Create backup directory
mkdir -p "$BACKUP_DIR"
# Check if database exists
if [ ! -f "$DB_PATH" ]; then
echo -e "${RED}✗ Database not found: $DB_PATH${NC}"
exit 1
fi
# Backup database
echo "Backing up database..."
cp "$DB_PATH" "$BACKUP_DIR/scheduler_${DATE}.db"
if [ $? -eq 0 ]; then
echo -e "${GREEN}✓ Database backed up: scheduler_${DATE}.db${NC}"
else
echo -e "${RED}✗ Database backup failed${NC}"
exit 1
fi
# Backup config if exists
if [ -f "$CONFIG_PATH" ]; then
echo "Backing up configuration..."
cp "$CONFIG_PATH" "$BACKUP_DIR/config_${DATE}.env"
if [ $? -eq 0 ]; then
echo -e "${GREEN}✓ Config backed up: config_${DATE}.env${NC}"
else
echo -e "${YELLOW}⚠ Config backup failed${NC}"
fi
fi
# Get database stats
DB_SIZE=$(du -h "$BACKUP_DIR/scheduler_${DATE}.db" | cut -f1)
JOB_COUNT=$(sqlite3 "$BACKUP_DIR/scheduler_${DATE}.db" "SELECT COUNT(*) FROM jobs;" 2>/dev/null || echo "N/A")
echo -e "Database size: ${DB_SIZE}, Jobs: ${JOB_COUNT}"
# Compress old backups (older than COMPRESS_AFTER_DAYS)
echo "Compressing old backups..."
COMPRESSED=$(find "$BACKUP_DIR" -name "*.db" -mtime +${COMPRESS_AFTER_DAYS} -exec gzip {} \; -print | wc -l)
if [ $COMPRESSED -gt 0 ]; then
echo -e "${GREEN}✓ Compressed $COMPRESSED old backup(s)${NC}"
fi
# Delete backups older than RETENTION_DAYS
echo "Cleaning up old backups..."
DELETED_DB=$(find "$BACKUP_DIR" -name "*.db.gz" -mtime +${RETENTION_DAYS} -delete -print | wc -l)
DELETED_CFG=$(find "$BACKUP_DIR" -name "*.env" -mtime +${RETENTION_DAYS} -delete -print | wc -l)
if [ $DELETED_DB -gt 0 ] || [ $DELETED_CFG -gt 0 ]; then
echo -e "${GREEN}✓ Deleted $DELETED_DB database(s) and $DELETED_CFG config(s)${NC}"
fi
# List recent backups
echo ""
echo "Recent backups:"
ls -lh "$BACKUP_DIR" | tail -5
# Optional: Upload to S3 or remote storage
# Uncomment and configure if needed
#
# if command -v aws &> /dev/null; then
# echo "Uploading to S3..."
# aws s3 cp "$BACKUP_DIR/scheduler_${DATE}.db" "s3://your-bucket/scheduler-backups/"
# echo -e "${GREEN}✓ Uploaded to S3${NC}"
# fi
echo ""
echo -e "${GREEN}=== Backup completed ===${NC}"
echo "Backup location: $BACKUP_DIR/scheduler_${DATE}.db"

173
scripts/monitor.sh Executable file
View File

@ -0,0 +1,173 @@
#!/bin/bash
#
# Scheduler Monitoring Script
# Checks service health, job status, and system resources
#
set -e
# Configuration
LOG_FILE="${LOG_FILE:-/var/log/scheduler-monitor.log}"
DB_PATH="${DB_PATH:-/opt/scheduler/scheduler.db}"
SERVICE_NAME="${SERVICE_NAME:-scheduler}"
ALERT_EMAIL="${ALERT_EMAIL:-}"
DISK_THRESHOLD=90
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
ALERTS=()
echo "=== Scheduler Monitor - $TIMESTAMP ===" | tee -a "$LOG_FILE"
# Check if service is running
check_service() {
if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
echo -e "${GREEN}✓ Service is running${NC}" | tee -a "$LOG_FILE"
return 0
elif docker ps --filter "name=movie_scheduler" --format "{{.Status}}" | grep -q "Up"; then
echo -e "${GREEN}✓ Docker container is running${NC}" | tee -a "$LOG_FILE"
return 0
else
echo -e "${RED}✗ Service is DOWN${NC}" | tee -a "$LOG_FILE"
ALERTS+=("Service is DOWN")
# Try to restart
echo "Attempting to restart service..." | tee -a "$LOG_FILE"
systemctl start "$SERVICE_NAME" 2>/dev/null || \
docker compose -f /opt/scheduler/docker-compose.prod.yml restart 2>/dev/null || true
return 1
fi
}
# Check database
check_database() {
if [ ! -f "$DB_PATH" ]; then
echo -e "${RED}✗ Database not found${NC}" | tee -a "$LOG_FILE"
ALERTS+=("Database file not found")
return 1
fi
# Database size
DB_SIZE=$(du -h "$DB_PATH" | cut -f1)
echo "Database size: $DB_SIZE" | tee -a "$LOG_FILE"
# Job counts
TOTAL=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs;" 2>/dev/null)
PENDING_PREP=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE prep_status='pending';" 2>/dev/null)
PENDING_PLAY=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE play_status='pending';" 2>/dev/null)
STREAMING=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE play_status='streaming';" 2>/dev/null)
FAILED=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM jobs WHERE prep_status='failed' OR play_status='failed';" 2>/dev/null)
echo "Jobs - Total: $TOTAL, Pending prep: $PENDING_PREP, Pending play: $PENDING_PLAY, Streaming: $STREAMING, Failed: $FAILED" | tee -a "$LOG_FILE"
if [ "$FAILED" -gt 0 ]; then
echo -e "${YELLOW}⚠ Found $FAILED failed job(s)${NC}" | tee -a "$LOG_FILE"
ALERTS+=("$FAILED failed jobs")
fi
if [ "$STREAMING" -gt 0 ]; then
echo -e "${GREEN}$STREAMING active stream(s)${NC}" | tee -a "$LOG_FILE"
fi
}
# Check disk space
check_disk() {
for PATH in "/mnt/storage" "/opt/scheduler" "/"; do
if [ -d "$PATH" ]; then
USAGE=$(df "$PATH" 2>/dev/null | tail -1 | awk '{print $5}' | sed 's/%//')
if [ -n "$USAGE" ]; then
if [ "$USAGE" -gt "$DISK_THRESHOLD" ]; then
echo -e "${RED}✗ Disk usage for $PATH: ${USAGE}%${NC}" | tee -a "$LOG_FILE"
ALERTS+=("Disk usage ${USAGE}% on $PATH")
else
echo -e "Disk usage for $PATH: ${USAGE}%" | tee -a "$LOG_FILE"
fi
fi
fi
done
}
# Check for stuck streams
check_stuck_streams() {
# Find streams that have been active for more than 4 hours
STUCK=$(sqlite3 "$DB_PATH" "
SELECT COUNT(*) FROM jobs
WHERE play_status='streaming'
AND datetime(stream_start_time) < datetime('now', '-4 hours')
" 2>/dev/null)
if [ "$STUCK" -gt 0 ]; then
echo -e "${YELLOW}⚠ Found $STUCK stream(s) active for >4 hours${NC}" | tee -a "$LOG_FILE"
ALERTS+=("$STUCK potentially stuck streams")
fi
}
# Check recent errors in logs
check_logs() {
if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
ERROR_COUNT=$(journalctl -u "$SERVICE_NAME" --since "5 minutes ago" 2>/dev/null | grep -i "ERROR" | wc -l)
else
ERROR_COUNT=$(docker compose -f /opt/scheduler/docker-compose.prod.yml logs --since 5m 2>/dev/null | grep -i "ERROR" | wc -l || echo "0")
fi
if [ "$ERROR_COUNT" -gt 10 ]; then
echo -e "${YELLOW}⚠ Found $ERROR_COUNT errors in last 5 minutes${NC}" | tee -a "$LOG_FILE"
ALERTS+=("$ERROR_COUNT recent errors")
fi
}
# Send alerts
send_alerts() {
if [ ${#ALERTS[@]} -gt 0 ]; then
echo "" | tee -a "$LOG_FILE"
echo -e "${RED}=== ALERTS ===${NC}" | tee -a "$LOG_FILE"
for alert in "${ALERTS[@]}"; do
echo "- $alert" | tee -a "$LOG_FILE"
done
# Send email if configured
if [ -n "$ALERT_EMAIL" ] && command -v mail &> /dev/null; then
{
echo "Scheduler Monitoring Alert - $TIMESTAMP"
echo ""
echo "The following issues were detected:"
for alert in "${ALERTS[@]}"; do
echo "- $alert"
done
echo ""
echo "Check $LOG_FILE for details"
} | mail -s "Scheduler Alert" "$ALERT_EMAIL"
fi
# Could also send to Slack, PagerDuty, etc.
# Example Slack webhook:
# if [ -n "$SLACK_WEBHOOK" ]; then
# curl -X POST -H 'Content-type: application/json' \
# --data "{\"text\":\"Scheduler Alert: ${ALERTS[*]}\"}" \
# "$SLACK_WEBHOOK"
# fi
fi
}
# Run checks
check_service
check_database
check_disk
check_stuck_streams
check_logs
send_alerts
echo "" | tee -a "$LOG_FILE"
# Exit with error if there are alerts
if [ ${#ALERTS[@]} -gt 0 ]; then
exit 1
fi
exit 0

238
scripts/setup-production.sh Executable file
View File

@ -0,0 +1,238 @@
#!/bin/bash
#
# Production Setup Script
# Quick setup for production deployment
#
set -e
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
echo -e "${BLUE}"
echo "=========================================="
echo " Movie Scheduler - Production Setup"
echo "=========================================="
echo -e "${NC}"
# Check if running as root
if [ "$EUID" -eq 0 ]; then
echo -e "${RED}Please don't run as root. Run as your regular user.${NC}"
echo "The script will use sudo when needed."
exit 1
fi
# Detect deployment method
echo "Choose deployment method:"
echo "1) Systemd service (bare metal)"
echo "2) Docker container"
read -p "Enter choice [1-2]: " DEPLOY_METHOD
if [ "$DEPLOY_METHOD" == "1" ]; then
echo -e "${GREEN}Setting up systemd service...${NC}"
# Check dependencies
echo "Checking dependencies..."
for cmd in python3 ffmpeg git sqlite3 systemctl; do
if ! command -v $cmd &> /dev/null; then
echo -e "${RED}✗ Missing: $cmd${NC}"
echo "Install it with: sudo apt-get install $cmd"
exit 1
else
echo -e "${GREEN}✓ Found: $cmd${NC}"
fi
done
# Check whisper.cpp
if ! command -v whisper.cpp &> /dev/null; then
echo -e "${YELLOW}⚠ whisper.cpp not found${NC}"
read -p "Install whisper.cpp now? [y/N]: " INSTALL_WHISPER
if [[ $INSTALL_WHISPER =~ ^[Yy]$ ]]; then
echo "Installing whisper.cpp..."
git clone https://github.com/ggerganov/whisper.cpp.git /tmp/whisper.cpp
cd /tmp/whisper.cpp
make
sudo cp main /usr/local/bin/whisper.cpp
sudo chmod +x /usr/local/bin/whisper.cpp
rm -rf /tmp/whisper.cpp
echo -e "${GREEN}✓ whisper.cpp installed${NC}"
fi
fi
# Create scheduler user
if ! id -u scheduler &> /dev/null; then
echo "Creating scheduler user..."
sudo useradd -r -s /bin/bash -d /opt/scheduler -m scheduler
sudo usermod -aG video,render scheduler
echo -e "${GREEN}✓ User created${NC}"
else
echo -e "${GREEN}✓ User already exists${NC}"
fi
# Create directories
echo "Creating directories..."
sudo mkdir -p /opt/scheduler /opt/models
sudo mkdir -p /mnt/storage/raw_movies /mnt/storage/final_movies
sudo chown scheduler:scheduler /opt/scheduler
sudo chown scheduler:scheduler /mnt/storage/raw_movies /mnt/storage/final_movies
# Copy application files
echo "Copying application files..."
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
sudo -u scheduler cp "$PROJECT_DIR/agent.py" /opt/scheduler/
sudo -u scheduler cp "$PROJECT_DIR/requirements.txt" /opt/scheduler/
# Create virtual environment
echo "Setting up Python virtual environment..."
sudo -u scheduler python3 -m venv /opt/scheduler/venv
sudo -u scheduler /opt/scheduler/venv/bin/pip install --upgrade pip
sudo -u scheduler /opt/scheduler/venv/bin/pip install -r /opt/scheduler/requirements.txt
# Download whisper model if needed
if [ ! -f "/opt/models/ggml-base.bin" ]; then
echo "Downloading whisper model..."
sudo wget -q -O /opt/models/ggml-base.bin \
https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin
echo -e "${GREEN}✓ Model downloaded${NC}"
fi
# Create config directory
echo "Setting up configuration..."
sudo mkdir -p /etc/scheduler
# Create environment file
if [ ! -f "/etc/scheduler/scheduler.env" ]; then
cat << 'EOF' | sudo tee /etc/scheduler/scheduler.env > /dev/null
# NocoDB Configuration
NOCODB_URL=https://your-nocodb.com/api/v2/tables/YOUR_TABLE_ID/records
NOCODB_TOKEN=YOUR_TOKEN_HERE
# RTMP Server
RTMP_SERVER=rtmp://your-rtmp-server.com/live/stream
# Storage Paths
RAW_DIR=/mnt/storage/raw_movies
FINAL_DIR=/mnt/storage/final_movies
# Whisper Model
WHISPER_MODEL=/opt/models/ggml-base.bin
# VAAPI Device
VAAPI_DEVICE=/dev/dri/renderD128
# Timing
NOCODB_SYNC_INTERVAL_SECONDS=60
WATCHDOG_CHECK_INTERVAL_SECONDS=10
STREAM_GRACE_PERIOD_MINUTES=15
EOF
sudo chmod 600 /etc/scheduler/scheduler.env
sudo chown scheduler:scheduler /etc/scheduler/scheduler.env
echo -e "${YELLOW}⚠ IMPORTANT: Edit /etc/scheduler/scheduler.env with your actual values${NC}"
echo " sudo nano /etc/scheduler/scheduler.env"
fi
# Install systemd service
echo "Installing systemd service..."
sudo cp "$PROJECT_DIR/scheduler.service" /etc/systemd/system/
sudo sed -i 's|^Environment=|# Environment=|g' /etc/systemd/system/scheduler.service
sudo sed -i 's|^# EnvironmentFile=|EnvironmentFile=|g' /etc/systemd/system/scheduler.service
sudo systemctl daemon-reload
echo ""
echo -e "${GREEN}=== Installation Complete ===${NC}"
echo ""
echo "Next steps:"
echo "1. Edit configuration:"
echo " sudo nano /etc/scheduler/scheduler.env"
echo ""
echo "2. Enable and start service:"
echo " sudo systemctl enable scheduler"
echo " sudo systemctl start scheduler"
echo ""
echo "3. Check status:"
echo " sudo systemctl status scheduler"
echo ""
echo "4. View logs:"
echo " sudo journalctl -u scheduler -f"
elif [ "$DEPLOY_METHOD" == "2" ]; then
echo -e "${GREEN}Setting up Docker deployment...${NC}"
# Check docker
if ! command -v docker &> /dev/null; then
echo -e "${RED}✗ Docker not found${NC}"
echo "Install Docker first: https://docs.docker.com/get-docker/"
exit 1
fi
# Check docker compose
if ! docker compose version &> /dev/null && ! docker-compose --version &> /dev/null; then
echo -e "${RED}✗ Docker Compose not found${NC}"
exit 1
fi
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
# Create deployment directory
read -p "Installation directory [/opt/scheduler]: " INSTALL_DIR
INSTALL_DIR=${INSTALL_DIR:-/opt/scheduler}
sudo mkdir -p "$INSTALL_DIR"
sudo chown $USER:$USER "$INSTALL_DIR"
# Copy files
echo "Copying files..."
cp "$PROJECT_DIR/agent.py" "$INSTALL_DIR/"
cp "$PROJECT_DIR/requirements.txt" "$INSTALL_DIR/"
cp "$PROJECT_DIR/Dockerfile" "$INSTALL_DIR/"
cp "$PROJECT_DIR/docker-compose.prod.yml" "$INSTALL_DIR/"
# Create environment file
if [ ! -f "$INSTALL_DIR/.env.production" ]; then
cp "$PROJECT_DIR/.env.production.example" "$INSTALL_DIR/.env.production"
echo -e "${YELLOW}⚠ Edit $INSTALL_DIR/.env.production with your actual values${NC}"
fi
# Create storage directories
sudo mkdir -p /mnt/storage/raw_movies /mnt/storage/final_movies
sudo mkdir -p /opt/models
# Download whisper model
if [ ! -f "/opt/models/ggml-base.bin" ]; then
echo "Downloading whisper model..."
sudo wget -q -O /opt/models/ggml-base.bin \
https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin
echo -e "${GREEN}✓ Model downloaded${NC}"
fi
echo ""
echo -e "${GREEN}=== Installation Complete ===${NC}"
echo ""
echo "Next steps:"
echo "1. Edit configuration:"
echo " nano $INSTALL_DIR/.env.production"
echo ""
echo "2. Build and start:"
echo " cd $INSTALL_DIR"
echo " docker compose -f docker-compose.prod.yml up -d"
echo ""
echo "3. View logs:"
echo " docker compose -f docker-compose.prod.yml logs -f"
echo ""
echo "4. Check status:"
echo " docker compose -f docker-compose.prod.yml ps"
else
echo -e "${RED}Invalid choice${NC}"
exit 1
fi
echo ""
echo -e "${BLUE}See PRODUCTION.md for complete documentation${NC}"

37
start-scheduler.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
set -e
echo "Starting Movie Scheduler..."
# Wait for config file from init container
CONFIG_FILE="/config/nocodb_config.env"
echo "Waiting for initialization to complete..."
for i in {1..60}; do
if [ -f "$CONFIG_FILE" ]; then
echo "✓ Configuration found!"
break
fi
if [ $i -eq 60 ]; then
echo "✗ Timeout waiting for configuration"
exit 1
fi
sleep 1
done
# Load configuration
source "$CONFIG_FILE"
# Set environment variables for the scheduler
export NOCODB_URL="http://nocodb:8080/api/v2/tables/${NOCODB_TABLE_ID}/records"
export NOCODB_TOKEN="${NOCODB_TOKEN}"
export RTMP_SERVER="${RTMP_SERVER:-rtmp://rtmp:1935/live/stream}"
echo "Configuration loaded:"
echo " NOCODB_URL: $NOCODB_URL"
echo " RTMP_SERVER: $RTMP_SERVER"
echo ""
echo "Starting scheduler agent..."
# Start the scheduler
exec python -u agent.py

119
start-test-environment.sh Executable file
View File

@ -0,0 +1,119 @@
#!/bin/bash
echo "=========================================="
echo " Movie Scheduler - Test Environment"
echo "=========================================="
echo ""
# Make scripts executable
chmod +x init-data.sh start-scheduler.sh 2>/dev/null || true
# Check if docker compose is available
if ! command -v docker &> /dev/null; then
echo "✗ Docker is not installed!"
echo "Please install Docker first: https://docs.docker.com/get-docker/"
exit 1
fi
if docker compose version &> /dev/null; then
COMPOSE_CMD="docker compose"
elif docker-compose --version &> /dev/null; then
COMPOSE_CMD="docker-compose"
else
echo "✗ Docker Compose is not installed!"
echo "Please install Docker Compose first"
exit 1
fi
echo "✓ Using: $COMPOSE_CMD"
echo ""
# Stop any existing containers
echo "Stopping any existing containers..."
$COMPOSE_CMD down 2>/dev/null || true
# Build and start all services
echo ""
echo "Building and starting all services..."
echo "This may take a few minutes on first run..."
echo ""
$COMPOSE_CMD up --build -d
echo ""
echo "=========================================="
echo " Waiting for services to initialize..."
echo "=========================================="
echo ""
# Wait for init to complete
echo "Waiting for initialization to complete (this may take 30-60 seconds)..."
sleep 5
# Follow init logs
echo ""
echo "--- Initialization Progress ---"
$COMPOSE_CMD logs -f init &
LOGS_PID=$!
# Wait for init to complete
while true; do
STATUS=$($COMPOSE_CMD ps init --format json 2>/dev/null | jq -r '.[0].State' 2>/dev/null || echo "unknown")
if [ "$STATUS" = "exited" ]; then
kill $LOGS_PID 2>/dev/null || true
wait $LOGS_PID 2>/dev/null || true
echo ""
echo "✓ Initialization completed!"
break
fi
sleep 2
done
echo ""
echo "=========================================="
echo " Test Environment Ready!"
echo "=========================================="
echo ""
echo "📺 NocoDB Dashboard:"
echo " URL: http://localhost:8080"
echo " Email: admin@test.com"
echo " Password: admin123"
echo ""
echo "📡 RTMP Server:"
echo " Stream URL: rtmp://localhost:1935/live/stream"
echo " Statistics: http://localhost:8081/stat"
echo ""
echo "🎬 Scheduler:"
echo " Status: Running"
echo " Database: ./scheduler.db"
echo ""
echo "=========================================="
echo " Useful Commands:"
echo "=========================================="
echo ""
echo "View scheduler logs:"
echo " $COMPOSE_CMD logs -f scheduler"
echo ""
echo "View all logs:"
echo " $COMPOSE_CMD logs -f"
echo ""
echo "Check service status:"
echo " $COMPOSE_CMD ps"
echo ""
echo "Stop all services:"
echo " $COMPOSE_CMD down"
echo ""
echo "Stop and remove all data:"
echo " $COMPOSE_CMD down -v"
echo ""
echo "=========================================="
echo ""
echo "The scheduler is now monitoring NocoDB for movies to process."
echo "A test movie 'BigBuckBunny' has been added and will be"
echo "processed 6 hours before its scheduled time."
echo ""
echo "To watch the stream, use a player like VLC:"
echo " vlc rtmp://localhost:1935/live/stream"
echo ""