Skip to content

Instantly share code, notes, and snippets.

@gphg
Created June 30, 2026 20:16
Show Gist options
  • Select an option

  • Save gphg/9b65081e2a0e313a45db50e663ed48b8 to your computer and use it in GitHub Desktop.

Select an option

Save gphg/9b65081e2a0e313a45db50e663ed48b8 to your computer and use it in GitHub Desktop.
Detect loops. This utilizes PySceneDetect and its predetected scenes.
#!/usr/bin/env python
# /// script
# dependencies = [
# "scenedetect[opencv]",
# ]
# ///
"""
LoopDetector: A utility to detect looping animations within specific scene segments
and process them into clips or GIFs.
This module detects seamless animation loops using multiple algorithms:
- 'mse' (default): Sliding Window Mean Squared Error — optimal for hand-drawn animation
held on consistent frames.
- 'histogram': Histogram chi-squared distance — robust to minor color shifts.
- 'perceptual': Perceptual hash distance (phash) — fast, scale/rotation tolerant.
Features:
- Multiple loop detection algorithms with pluggable architecture
- Crop region support for isolating subject focus (non-moving backgrounds)
- Automatic frame downscaling for detection (detection-only, not affecting exports)
- Persistent CSV caching with frame-accurate metadata
- Crop coordinates stored as critical metadata for downstream transparency workflows
- Multi-candidate preview generation with indexed filenames
- Enforced minimum output duration (≥ 1.0s) via streaming loop
Workflow alignment:
This implementation strictly follows the "PreCure Production Pipeline: Scene & Loop
Extraction Guide" (Stage 2: Loop Detection Workflow):
- Cache filenames include start frame: {file_base}_f{start_fc}_loops_{detection_method}.csv
- Preview filenames map to indices: {file_base}_f{start_fc}_Loop-{num}.mp4
- Crop coordinates persisted to CSV as critical metadata
- Multi-candidate export with explicit index tracking
"""
from typing import Optional, Tuple, List, Dict
import os
import sys
import csv
import tempfile
import argparse
import shutil
import subprocess
import math
import re
from abc import ABC, abstractmethod
import cv2
import numpy as np
from scenedetect import FrameTimecode
# =========================
# Exceptions
# =========================
class LoopError(Exception):
pass
class LoopInputError(LoopError):
pass
class LoopDetectionError(LoopError):
pass
class LoopProcessingError(LoopError):
pass
# =========================
# Cache (CSV) with Crop Metadata
# =========================
class LoopCache:
"""
Manages persistent caching of detected loop metadata in CSV format.
Cache filename: {file_base}_f{start_fc}_loops_{detection_method}.csv
This naming scheme ensures:
- Sequential runs on the same episode don't overwrite existing data
- Start frame is explicitly tracked for frame-accurate reproducibility
- Detection method is encoded for multi-method comparison
Persisted metadata includes:
- Loop timecodes and frame counts
- Detection quality metrics
- Crop coordinates as CRITICAL metadata for downstream transparency workflows
"""
def __init__(self, output_dir: str, file_base: str, start_fc: int, detection_method: str):
self.output_dir = output_dir
self.file_base = file_base
self.start_fc = start_fc
self.detection_method = detection_method
def get_cache_path(self) -> str:
"""Return cache file path with start_fc and detection_method encoded."""
return os.path.join(
self.output_dir,
f"{self.file_base}_f{self.start_fc}_loops_{self.detection_method}.csv"
)
def save(self, data: Dict):
"""
Save loop metadata atomically to CSV.
Columns (in order):
num, start_tc, end_tc, start_fc, end_fc, duration, frame_count,
similarity, quality, fps, sample_step, detection_method,
crop_x, crop_y, crop_w, crop_h
Crop coordinates are stored for critical downstream transparency workflows.
"""
cache_file = self.get_cache_path()
keys = [
"num",
"start_tc",
"end_tc",
"start_fc",
"end_fc",
"duration",
"frame_count",
"similarity",
"quality",
"fps",
"sample_step",
"detection_method",
"crop_x",
"crop_y",
"crop_w",
"crop_h",
]
fd, tmp_path = tempfile.mkstemp(dir=self.output_dir, text=True)
try:
with os.fdopen(fd, "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=keys)
writer.writeheader()
row = {k: data.get(k, "") for k in keys}
writer.writerow(row)
os.replace(tmp_path, cache_file)
print(f"[INFO] Loop cache saved to {cache_file}")
except Exception as e:
if os.path.exists(tmp_path):
os.remove(tmp_path)
raise LoopError(f"Failed to save cache: {e}")
def load(self) -> Optional[Dict]:
"""Load cached loop metadata. Returns None if cache doesn't exist."""
cache_file = self.get_cache_path()
if not os.path.exists(cache_file):
return None
try:
with open(cache_file, "r", newline="") as f:
reader = csv.DictReader(f)
row = next(reader, None)
if row is None:
return None
# Parse crop coordinates as critical metadata
crop_x = int(row.get("crop_x", 0)) if row.get("crop_x") else 0
crop_y = int(row.get("crop_y", 0)) if row.get("crop_y") else 0
crop_w = int(row.get("crop_w", 0)) if row.get("crop_w") else 0
crop_h = int(row.get("crop_h", 0)) if row.get("crop_h") else 0
crop = (crop_x, crop_y, crop_w, crop_h) if any([crop_x, crop_y, crop_w, crop_h]) else None
result = {
"num": int(row.get("num", 1)),
"start_tc": row.get("start_tc", ""),
"end_tc": row.get("end_tc", ""),
"start_fc": int(row.get("start_fc", 0)),
"end_fc": int(row.get("end_fc", 0)),
"duration": float(row.get("duration", 0.0)),
"frame_count": int(row.get("frame_count", 0)),
"similarity": float(row.get("similarity", 0.0)),
"quality": float(row.get("quality", 0.0)),
"fps": float(row.get("fps", 0.0)),
"sample_step": int(row.get("sample_step", 1)),
"detection_method": row.get("detection_method", "unknown"),
"crop": crop,
}
return result
except Exception as e:
print(f"[WARN] Failed to load cache: {e}", file=sys.stderr)
return None
# =========================
# Loop Detection Algorithms (Pluggable)
# =========================
class LoopDetectionAlgorithm(ABC):
"""Base class for loop detection algorithms."""
@abstractmethod
def name(self) -> str:
"""Return algorithm name for logging/caching."""
pass
@abstractmethod
def compute_similarity(self, img_a: np.ndarray, img_b: np.ndarray) -> float:
"""
Compute similarity between two frames.
Returns:
Scalar value where LOWER = more similar (like MSE/distance).
Implementations should normalize to ~0-255 scale for cross-algorithm comparison.
"""
pass
@abstractmethod
def get_default_threshold(self) -> float:
"""Return default sensitivity threshold for this algorithm."""
pass
class MSELoopDetector(LoopDetectionAlgorithm):
"""
Sliding Window Mean Squared Error (MSE) — default algorithm.
Optimal for hand-drawn animation with consistent held frames (animating on 2s/3s).
Uses grayscale conversion to reduce noise from color variations.
Lower MSE = more similar frames.
"""
def name(self) -> str:
return "mse"
def get_default_threshold(self) -> float:
# MSE on 0-255 scale: ~100-200 is "very similar" for animation
return 150.0
def compute_similarity(self, img_a: np.ndarray, img_b: np.ndarray) -> float:
"""Mean Squared Error on grayscale images."""
if img_a is None or img_b is None:
return float("inf")
if img_a.shape != img_b.shape:
img_b = cv2.resize(img_b, (img_a.shape[1], img_a.shape[0]),
interpolation=cv2.INTER_AREA)
# Convert to grayscale
if img_a.ndim == 3:
a = cv2.cvtColor(img_a, cv2.COLOR_BGR2GRAY)
else:
a = img_a
if img_b.ndim == 3:
b = cv2.cvtColor(img_b, cv2.COLOR_BGR2GRAY)
else:
b = img_b
err = np.mean((a.astype("float32") - b.astype("float32")) ** 2)
return float(err)
class HistogramLoopDetector(LoopDetectionAlgorithm):
"""
Histogram chi-squared distance.
Robust to minor color shifts and lighting changes.
Faster than MSE; trades frame-level precision for robustness.
"""
def name(self) -> str:
return "histogram"
def get_default_threshold(self) -> float:
# Chi-squared distance on normalized histograms: lower threshold = stricter
return 50.0
def compute_similarity(self, img_a: np.ndarray, img_b: np.ndarray) -> float:
"""Chi-squared distance between histograms."""
if img_a is None or img_b is None:
return float("inf")
if img_a.shape != img_b.shape:
img_b = cv2.resize(img_b, (img_a.shape[1], img_a.shape[0]),
interpolation=cv2.INTER_AREA)
# Convert to HSV for better color separation
if img_a.ndim == 3:
hsv_a = cv2.cvtColor(img_a, cv2.COLOR_BGR2HSV)
else:
hsv_a = cv2.cvtColor(cv2.cvtColor(img_a, cv2.COLOR_GRAY2BGR), cv2.COLOR_BGR2HSV)
if img_b.ndim == 3:
hsv_b = cv2.cvtColor(img_b, cv2.COLOR_BGR2HSV)
else:
hsv_b = cv2.cvtColor(cv2.cvtColor(img_b, cv2.COLOR_GRAY2BGR), cv2.COLOR_BGR2HSV)
# Compute histograms (H, S, V channels)
hist_a = cv2.calcHist([hsv_a], [0, 1], None, [180, 256], [0, 180, 0, 256])
hist_b = cv2.calcHist([hsv_b], [0, 1], None, [180, 256], [0, 180, 0, 256])
# Normalize
cv2.normalize(hist_a, hist_a, alpha=1, beta=0, norm_type=cv2.NORM_L1)
cv2.normalize(hist_b, hist_b, alpha=1, beta=0, norm_type=cv2.NORM_L1)
# Chi-squared distance
distance = cv2.compareHist(hist_a, hist_b, cv2.HISTCMP_CHISQR)
return float(distance)
class PerceptualHashLoopDetector(LoopDetectionAlgorithm):
"""
Perceptual Hash (pHash) distance.
Fast, scale/rotation tolerant. Uses DCT to extract perceptual features.
Good for quick pre-filtering; less precise than MSE.
"""
def name(self) -> str:
return "perceptual"
def get_default_threshold(self) -> float:
# Hamming distance on 64-bit hashes: 0-16 bits set difference is "very similar"
# Scaled to ~0-255: threshold ~50
return 50.0
def _compute_phash(self, img: np.ndarray) -> int:
"""Compute 64-bit perceptual hash using DCT."""
if img is None:
return 0
# Resize to 8x8 and convert to grayscale
if img.ndim == 3:
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
else:
gray = img
small = cv2.resize(gray, (8, 8), interpolation=cv2.INTER_AREA)
# Apply DCT
dct = cv2.dct(np.float32(small))
# Use top-left 8x8 (already 8x8, so all of it)
# Compute average, then hash based on > average
avg = np.mean(dct)
hash_bits = 0
for i in range(8):
for j in range(8):
if dct[i, j] > avg:
hash_bits = (hash_bits << 1) | 1
else:
hash_bits = hash_bits << 1
return hash_bits
def compute_similarity(self, img_a: np.ndarray, img_b: np.ndarray) -> float:
"""Hamming distance between perceptual hashes."""
hash_a = self._compute_phash(img_a)
hash_b = self._compute_phash(img_b)
# Count differing bits (Hamming distance)
xor = hash_a ^ hash_b
distance = bin(xor).count('1')
# Scale to ~0-255 range (max 64 bits, so 64 -> 255)
return float(distance * 4)
class LoopDetectorFactory:
"""Factory for creating detection algorithm instances."""
ALGORITHMS = {
"mse": MSELoopDetector,
"histogram": HistogramLoopDetector,
"perceptual": PerceptualHashLoopDetector,
}
@staticmethod
def create(method: str) -> LoopDetectionAlgorithm:
"""
Create a detector instance by method name.
Args:
method: Algorithm name ('mse', 'histogram', 'perceptual')
Returns:
Configured LoopDetectionAlgorithm instance
Raises:
LoopInputError: If method is unknown
"""
if method not in LoopDetectorFactory.ALGORITHMS:
available = ", ".join(LoopDetectorFactory.ALGORITHMS.keys())
raise LoopInputError(
f"Unknown detection method: {method}. "
f"Available: {available}"
)
return LoopDetectorFactory.ALGORITHMS[method]()
@staticmethod
def list_methods() -> List[str]:
"""Return list of available method names."""
return list(LoopDetectorFactory.ALGORITHMS.keys())
@staticmethod
def get_default_threshold(method: str) -> float:
"""Get default threshold for a method."""
algo = LoopDetectorFactory.create(method)
return algo.get_default_threshold()
# =========================
# Detection helpers
# =========================
def _to_frame(tc_str: str, fps: float) -> int:
"""
Convert a user-provided time spec to a frame index.
Prefer interpreting purely numeric strings as frame indices,
otherwise use FrameTimecode.
"""
if tc_str is None:
raise ValueError("Timecode string is None")
s = str(tc_str).strip()
# numeric integer -> frames
if re.match(r"^\d+$", s):
return int(s)
# percentage or floats are not frames, fallback to FrameTimecode
try:
ft = FrameTimecode(s, fps=fps)
return ft.frame_num
except Exception as e:
# last resort: try int conversion (float-as-int)
try:
return int(float(s))
except Exception:
raise ValueError(f"Invalid timecode or frame '{tc_str}': {e}")
def _frame_to_tc(frame: int, fps: float) -> str:
return FrameTimecode(frame, fps=fps).get_timecode()
def _parse_downscale_arg(arg: Optional[str], region_w: int, region_h: int) -> Optional[Tuple[int, int]]:
"""
Parse the --downscale argument and return target (w, h) or None for no scaling.
Supported arg forms:
- None or 'auto' -> automatic heuristic (small video widths are left unchanged)
- '480' -> width 480, height computed to keep aspect ratio, height forced even
- '480x270' -> width x height exact
- '50%' -> percentage of region size
The function ensures resulting height is even.
"""
if arg is None:
arg = "auto"
s = str(arg).strip().lower()
if s == "auto":
# Heuristic: reduce large regions for detection to reasonable widths
if region_w > 1280:
target_w = 640
elif region_w > 800:
target_w = 480
elif region_w > 640:
target_w = 480
else:
return None
target_h = int(round(target_w * (region_h / region_w)))
elif s.endswith("%"):
try:
pct = float(s.rstrip("%")) / 100.0
if pct <= 0 or pct > 1e6:
return None
target_w = max(1, int(round(region_w * pct)))
target_h = max(1, int(round(region_h * pct)))
except Exception:
return None
elif "x" in s:
try:
parts = s.split("x")
target_w = int(parts[0])
target_h = int(parts[1])
except Exception:
return None
else:
# try parse integer width
try:
target_w = int(s)
# compute height keeping aspect ratio
target_h = int(round(target_w * (region_h / region_w)))
except Exception:
return None
# Make sure width/height positive and height even
target_w = max(1, int(target_w))
target_h = max(1, int(target_h))
if target_h % 2 == 1:
target_h -= 1
if target_h < 2:
target_h = 2
return (target_w, target_h)
# =========================
# Detector
# =========================
class LoopDetector:
"""
Detects a loop within a start..end time segment of a video.
Detection strategy:
- Read sampled frames from range [start_frame, end_frame)
- Use a small reference block from start of the range
- Slide over second half of the frames to find position with lowest avg similarity
- Return metadata describing loop (start frame, end frame, quality, etc.)
Downscaling:
- Detection frames can be downscaled for performance (crop applied first).
- Downscaling is detection-only and does not affect exported outputs.
Algorithm:
- Pluggable detection algorithm (mse, histogram, perceptual)
- Each algorithm computes frame-pair similarity differently
"""
def __init__(
self,
algorithm: LoopDetectionAlgorithm,
max_sample_frames: int = 1000,
ref_block_size: int = 5,
):
self.algorithm = algorithm
self.max_sample_frames = int(max_sample_frames)
self.ref_block_size = int(ref_block_size)
def _open_capture(self, path: str):
cap = cv2.VideoCapture(path)
if not cap.isOpened():
raise LoopDetectionError(f"Cannot open video file: {path}")
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
return cap, float(fps), total, width, height
def _read_frames_with_indices(
self,
cap: cv2.VideoCapture,
start_frame: int,
end_frame: int,
step: int,
crop: Optional[Tuple[int, int, int, int]] = None,
downscale_target: Optional[Tuple[int, int]] = None,
) -> List[Tuple[int, np.ndarray]]:
"""
Read sampled frames and return list of tuples: (absolute_frame_index, frame).
Applies crop first (if provided) then downscale (if provided).
"""
frames = []
cur = start_frame
cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
while cur < end_frame:
ret, frame = cap.read()
if not ret:
break
if crop:
x, y, w, h = crop
# guard crop bounds
h_total, w_total = frame.shape[:2]
x2 = min(w_total, x + w)
y2 = min(h_total, y + h)
x = max(0, x)
y = max(0, y)
if x >= x2 or y >= y2:
# invalid crop, return empty to indicate failure
return []
frame = frame[y:y2, x:x2]
if downscale_target:
tw, th = downscale_target
# apply resizing
frame = cv2.resize(frame, (tw, th), interpolation=cv2.INTER_AREA)
frames.append((cur, frame))
cur += step
if step > 1:
cap.set(cv2.CAP_PROP_POS_FRAMES, cur)
return frames
def detect(
self,
video_path: str,
start_tc: str,
end_tc: str,
crop: Optional[Tuple[int, int, int, int]] = None,
downscale_arg: Optional[str] = "auto",
) -> Dict:
"""
Detect loop in video segment.
Args:
video_path: Path to video file
start_tc: Start timecode (frame or HH:MM:SS.mmm)
end_tc: End timecode (frame or HH:MM:SS.mmm)
crop: Optional crop bounds (x, y, w, h)
downscale_arg: Downscale argument ('auto', '480', '480x270', '50%')
Returns:
Dict with keys: num, start_tc, end_tc, start_fc, end_fc, duration,
frame_count, similarity, quality, fps, sample_step,
detection_method, crop_x, crop_y, crop_w, crop_h
"""
if not os.path.exists(video_path):
raise LoopInputError(f"Input file not found: {video_path}")
cap, fps, total_frames, vid_w, vid_h = self._open_capture(video_path)
try:
start_frame = _to_frame(start_tc, fps)
end_frame = _to_frame(end_tc, fps)
except ValueError as e:
cap.release()
raise LoopInputError(str(e))
# Clamp bounds
start_frame = max(0, min(start_frame, max(0, total_frames - 1)))
end_frame = max(0, min(end_frame, total_frames))
if end_frame <= start_frame + 1:
cap.release()
raise LoopInputError("Range too short for detection")
seg_count = end_frame - start_frame
# Decide sampling step
step = 1
if seg_count > self.max_sample_frames:
step = math.ceil(seg_count / self.max_sample_frames)
# Compute cropped region size for downscale decision
if crop:
x, y, w, h = crop
region_w = max(1, min(w, vid_w - x))
region_h = max(1, min(h, vid_h - y))
else:
region_w = vid_w
region_h = vid_h
# Determine downscale target based on argument and region size
downscale_target = _parse_downscale_arg(downscale_arg, region_w, region_h)
if downscale_target:
print(f"[INFO] Detection downscale target: {downscale_target[0]}x{downscale_target[1]} (after crop)")
else:
print("[INFO] No downscaling will be applied for detection frames")
sampled = self._read_frames_with_indices(cap, start_frame, end_frame, step, crop, downscale_target)
cap.release()
if len(sampled) < 2:
raise LoopDetectionError("Not enough frames read for loop detection")
abs_indices = [t[0] for t in sampled]
frames = [t[1] for t in sampled]
# choose small reference block from start
ref_block = min(self.ref_block_size, max(1, len(frames) // 20))
ref_frames = frames[:ref_block]
# Search region: latter half
search_start_idx = max(len(frames) // 2, ref_block)
best_score = float("inf")
best_sampled_idx = len(frames) - 1
for j in range(search_start_idx + ref_block - 1, len(frames)):
candidate_start = j - ref_block + 1
if candidate_start < 0:
continue
scores = []
for k in range(ref_block):
a = ref_frames[k]
b = frames[candidate_start + k]
score = self.algorithm.compute_similarity(a, b)
scores.append(score)
avg = float(np.mean(scores))
if avg < best_score:
best_score = avg
best_sampled_idx = j
approx_abs_end = abs_indices[best_sampled_idx]
approx_abs_end = min(approx_abs_end, end_frame - 1)
loop_start_frame = start_frame
loop_end_frame = approx_abs_end
frame_count = loop_end_frame - loop_start_frame
duration = frame_count / fps if fps > 0 else 0.0
# Normalize quality score (algorithm-agnostic)
# Assume score range ~0-255; quality = 1 - (score / 255)
max_score = 255.0
quality = 1.0 - (best_score / max_score)
quality = max(0.0, min(1.0, quality))
result = {
"num": 1,
"start_tc": _frame_to_tc(loop_start_frame, fps),
"end_tc": _frame_to_tc(loop_end_frame, fps),
"start_fc": int(loop_start_frame),
"end_fc": int(loop_end_frame),
"duration": float(duration),
"frame_count": int(frame_count),
"similarity": float(best_score),
"quality": float(quality),
"fps": float(fps),
"sample_step": int(step),
"detection_method": self.algorithm.name(),
# Crop coordinates as critical metadata
"crop_x": int(crop[0]) if crop else 0,
"crop_y": int(crop[1]) if crop else 0,
"crop_w": int(crop[2]) if crop else 0,
"crop_h": int(crop[3]) if crop else 0,
}
return result
# =========================
# Processor (FFmpeg) with minimum 1-second loop behavior
# =========================
class LoopProcessor:
"""
Exports detected loop to video (mp4/webm) or gif using ffmpeg.
Ensures final exported clip is at least 1.0 second long by looping
output-only when needed.
"""
MIN_OUTPUT_SECONDS = 1.0
def __init__(self):
if not shutil.which("ffmpeg"):
raise LoopProcessingError("ffmpeg not found in PATH")
def _run(self, cmd: List[str]) -> subprocess.CompletedProcess:
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return proc
def _ensure_long_enough_by_stream_loop(self, short_clip_path: str, out_path: str, duration_secs: float):
try:
probe_cmd = [
"ffprobe",
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
short_clip_path,
]
p = subprocess.run(probe_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.returncode == 0:
clip_dur = float(p.stdout.decode().strip() or 0.0)
else:
clip_dur = None
except Exception:
clip_dur = None
if not clip_dur or clip_dur <= 0:
raise LoopProcessingError("Could not determine temporary clip duration for looping")
if clip_dur >= duration_secs:
shutil.move(short_clip_path, out_path)
return
plays_needed = math.ceil(duration_secs / clip_dur)
loops_needed = max(0, plays_needed - 1)
cmd = ["ffmpeg", "-y", "-stream_loop", str(loops_needed), "-i", short_clip_path, "-t", f"{duration_secs:.3f}", "-c", "copy", out_path]
proc = self._run(cmd)
if proc.returncode != 0:
err = proc.stderr.decode("utf-8", errors="ignore")
raise LoopProcessingError(f"ffmpeg looping failed: {err}")
try:
if os.path.exists(short_clip_path):
os.remove(short_clip_path)
except Exception:
pass
def generate_crop_preview(self, input_path: str, metadata: Dict, crop: Tuple[int, int, int, int], output_path: str):
"""Generates a short video preview with a drawn rectangle showing the crop area."""
start_frame = int(metadata["start_fc"])
end_frame = int(metadata["end_fc"])
fps = float(metadata.get("fps", 30.0))
start_secs = start_frame / fps
clip_duration = max(0.001, (end_frame - start_frame) / fps)
# Clamp preview duration to max 10.0 seconds, or the clip's actual duration
duration = min(10.0, clip_duration)
x, y, w, h = crop
vf = f"drawbox=x={x}:y={y}:w={w}:h={h}:color=red@0.8:thickness=4"
cmd = [
"ffmpeg", "-y",
"-ss", f"{start_secs:.3f}",
"-i", input_path,
"-t", f"{duration:.3f}",
"-vf", vf,
"-c:v", "libx264", "-preset", "ultrafast", "-crf", "28",
"-an", # Strip audio for a faster preview export
output_path
]
proc = self._run(cmd)
if proc.returncode != 0:
stderr = proc.stderr.decode("utf-8", errors="ignore")
raise LoopProcessingError(f"ffmpeg drawbox failed: {stderr}")
def export_video(self, input_path: str, metadata: Dict, output_path: str, mode: str = "normal", scale: Optional[str] = None):
start_frame = int(metadata["start_fc"])
end_frame = int(metadata["end_fc"])
fps = float(metadata.get("fps", 30.0))
clip_duration = max(0.001, (end_frame - start_frame) / fps)
start_secs = start_frame / fps
tmp_dir = tempfile.mkdtemp(prefix="loop_tmp_")
tmp_clip = os.path.join(tmp_dir, "loop_clip.mp4")
ss = ["-ss", f"{start_secs:.3f}"]
input_arg = ["-i", input_path]
t = ["-t", f"{clip_duration:.3f}"]
if mode == "copy":
cmd = ["ffmpeg", "-y"] + ss + input_arg + t + ["-c", "copy", tmp_clip]
else:
preset = "ultrafast" if mode == "prototype" else "medium"
crf = "32" if mode == "prototype" else "22"
vf = None
if scale:
clean_scale = scale.replace("-1", "-2")
vf = f"scale={clean_scale}"
cmd = ["ffmpeg", "-y"] + ss + input_arg + t + ["-c:v", "libx264", "-preset", preset, "-crf", crf, "-c:a", "aac"]
if vf:
cmd += ["-vf", vf]
cmd += [tmp_clip]
proc = self._run(cmd)
if proc.returncode != 0:
stderr = proc.stderr.decode("utf-8", errors="ignore")
try:
if os.path.exists(tmp_clip):
os.remove(tmp_clip)
except Exception:
pass
raise LoopProcessingError(f"ffmpeg failed to create temp clip: {stderr}")
desired = max(self.MIN_OUTPUT_SECONDS, clip_duration)
if clip_duration >= self.MIN_OUTPUT_SECONDS:
try:
shutil.move(tmp_clip, output_path)
except Exception:
shutil.copy(tmp_clip, output_path)
os.remove(tmp_clip)
try:
os.rmdir(tmp_dir)
except Exception:
pass
print(f"[INFO] Exported video to {output_path}")
return
else:
try:
self._ensure_parent_dir(output_path)
self._ensure_long_enough_by_stream_loop(tmp_clip, output_path, self.MIN_OUTPUT_SECONDS)
finally:
try:
os.rmdir(tmp_dir)
except Exception:
pass
print(f"[INFO] Exported video (looped to >={self.MIN_OUTPUT_SECONDS}s) to {output_path}")
return
def export_gif(self, input_path: str, metadata: Dict, output_path: str, scale: Optional[str] = None):
start_frame = int(metadata["start_fc"])
end_frame = int(metadata["end_fc"])
fps = float(metadata.get("fps", 30.0))
clip_duration = max(0.001, (end_frame - start_frame) / fps)
start_secs = start_frame / fps
tmp_dir = tempfile.mkdtemp(prefix="loop_tmp_")
tmp_clip_mp4 = os.path.join(tmp_dir, "loop_clip.mp4")
tmp_long_mp4 = os.path.join(tmp_dir, "loop_long.mp4")
ss = ["-ss", f"{start_secs:.3f}"]
input_arg = ["-i", input_path]
t = ["-t", f"{clip_duration:.3f}"]
cmd = ["ffmpeg", "-y"] + ss + input_arg + t + ["-c:v", "libx264", "-preset", "medium", "-crf", "22", "-c:a", "aac", tmp_clip_mp4]
proc = self._run(cmd)
if proc.returncode != 0:
stderr = proc.stderr.decode("utf-8", errors="ignore")
try:
if os.path.exists(tmp_clip_mp4):
os.remove(tmp_clip_mp4)
except Exception:
pass
raise LoopProcessingError(f"ffmpeg failed to create temp mp4 for GIF: {stderr}")
if clip_duration >= self.MIN_OUTPUT_SECONDS:
tmp_for_gif = tmp_clip_mp4
else:
desired = self.MIN_OUTPUT_SECONDS
try:
self._ensure_long_enough_by_stream_loop(tmp_clip_mp4, tmp_long_mp4, desired)
except LoopProcessingError as e:
try:
if os.path.exists(tmp_clip_mp4):
os.remove(tmp_clip_mp4)
except Exception:
pass
raise
tmp_for_gif = tmp_long_mp4
palette_file = os.path.join(tmp_dir, "palette.png")
vf_parts = []
if scale:
clean_scale = scale.replace("-1", "-2")
vf_parts.append(f"scale={clean_scale}")
gif_fps = min(15, max(5, int(round(fps))))
vf_parts.append(f"fps={gif_fps}")
vf_filter = ",".join(vf_parts) if vf_parts else f"fps={gif_fps}"
palette_cmd = [
"ffmpeg",
"-y",
"-i",
tmp_for_gif,
"-vf",
f"{vf_filter},palettegen",
"-frames:v",
"1",
palette_file,
]
p1 = self._run(palette_cmd)
if p1.returncode != 0:
stderr = p1.stderr.decode("utf-8", errors="ignore")
for f in (tmp_clip_mp4, tmp_long_mp4, palette_file):
try:
if os.path.exists(f):
os.remove(f)
except Exception:
pass
raise LoopProcessingError(f"ffmpeg palettegen failed: {stderr}")
gif_cmd = [
"ffmpeg",
"-y",
"-i",
tmp_for_gif,
"-i",
palette_file,
"-lavfi",
f"{vf_filter} [x]; [x][1:v] paletteuse",
output_path,
]
p2 = self._run(gif_cmd)
try:
if os.path.exists(palette_file):
os.remove(palette_file)
if os.path.exists(tmp_clip_mp4) and tmp_clip_mp4 != tmp_for_gif:
os.remove(tmp_clip_mp4)
if os.path.exists(tmp_long_mp4):
try:
os.remove(tmp_long_mp4)
except Exception:
pass
try:
os.rmdir(tmp_dir)
except Exception:
pass
except Exception:
pass
if p2.returncode != 0:
stderr = p2.stderr.decode("utf-8", errors="ignore")
raise LoopProcessingError(f"ffmpeg gif generation failed: {stderr}")
print(f"[INFO] Exported GIF to {output_path}")
@staticmethod
def _ensure_parent_dir(path: str):
d = os.path.dirname(path)
if d and not os.path.exists(d):
os.makedirs(d, exist_ok=True)
# =========================
# Orchestrator
# =========================
class LoopWrapper:
"""
High-level orchestrator: detection, caching, and processing.
Strictly follows Stage 2 workflow from "PreCure Production Pipeline" guide:
- Cache filenames include start frame: {file_base}_f{start_fc}_loops_{detection_method}.csv
- Crop coordinates stored as critical metadata for transparency workflows
- Multi-candidate previews with indexed filenames: {file_base}_f{start_fc}_Loop-{num}.mp4
"""
def __init__(
self,
input_path: str,
output_dir: str,
detection_method: str = "mse",
max_sample_frames: int = 1000,
ref_block_size: int = 5,
):
if not os.path.isfile(input_path):
raise LoopInputError(f"Input video not found: {input_path}")
self.input_path = input_path
self.output_dir = output_dir
self.detection_method = detection_method
self.filename = os.path.basename(input_path)
self.file_base, self.extension = os.path.splitext(self.filename)
try:
if not os.path.exists(self.output_dir):
os.makedirs(self.output_dir)
except Exception as e:
raise LoopInputError(f"Could not create output directory {self.output_dir}: {e}")
# Create detection algorithm instance
try:
self.algorithm = LoopDetectorFactory.create(detection_method)
except LoopInputError as e:
raise LoopInputError(f"Failed to initialize detector: {e}")
self.detector = LoopDetector(
self.algorithm,
max_sample_frames=max_sample_frames,
ref_block_size=ref_block_size,
)
self.processor = LoopProcessor()
# Cache will be initialized once we know start_fc
self.cache = None
def detect_loop(
self,
start: str,
end: str,
crop: Optional[Tuple[int, int, int, int]] = None,
downscale_arg: Optional[str] = "auto",
use_cache: bool = True,
) -> Dict:
"""
Detect loop within scene segment.
Args:
start: Start timecode/frame
end: End timecode/frame
crop: Optional crop bounds (x, y, w, h)
downscale_arg: Downscale argument for detection
use_cache: Whether to use cached results
Returns:
Dict with loop metadata including crop coordinates
"""
# Get FPS to convert start to frame count
cap = cv2.VideoCapture(self.input_path)
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
cap.release()
try:
start_fc = _to_frame(start, fps)
except ValueError as e:
raise LoopInputError(f"Could not parse start timecode: {e}")
# Initialize cache with start_fc
self.cache = LoopCache(self.output_dir, self.file_base, start_fc, self.detection_method)
if use_cache:
cached = self.cache.load()
if cached:
try:
req_end_fc = _to_frame(end, cached.get("fps", fps))
if start_fc == cached.get("start_fc") and req_end_fc == cached.get("end_fc"):
print(f"[INFO] Loaded cached loop metadata from {self.cache.get_cache_path()}")
return cached
else:
print("[INFO] Cache found but range differs; ignoring cache", file=sys.stderr)
except Exception:
print("[INFO] Cache present but could not validate range; re-detecting", file=sys.stderr)
print(f"[INFO] Detecting loop using '{self.detection_method}' algorithm in range {start} -> {end} ...")
try:
metadata = self.detector.detect(self.input_path, start, end, crop, downscale_arg)
except LoopInputError:
raise
except LoopDetectionError:
raise
except Exception as e:
raise LoopDetectionError(f"Loop detection failed: {e}")
try:
self.cache.save(metadata)
except LoopError as e:
print(f"[WARN] Failed to write cache: {e}", file=sys.stderr)
return metadata
def process(
self,
metadata: Dict,
process_type: Optional[str],
mode: str,
scale: Optional[str],
out_name: Optional[str],
dry_run: bool = False,
crop: Optional[Tuple[int, int, int, int]] = None,
) -> List[str]:
"""
Export detected loop to video/GIF.
Args:
metadata: Loop metadata from detect_loop
process_type: Export type ('video', 'gif', or None)
mode: Encoding mode ('normal', 'prototype', 'copy')
scale: Optional scaling
out_name: Custom output filename
dry_run: Show what would be exported without doing it
crop: Crop bounds (for preview generation)
Returns:
List of produced file paths
"""
produced = []
if process_type is None and not dry_run:
return produced
start_fc = int(metadata.get("start_fc", 0))
if out_name:
out_basename = out_name
else:
ext = ".gif" if process_type == "gif" else ".mp4"
out_basename = f"{self.file_base}_f{start_fc}_Loop-001{ext}"
out_path = os.path.join(self.output_dir, out_basename)
if dry_run:
if process_type:
print(f"[DRY-RUN] Would export {process_type} to {out_path} (mode={mode}, scale={scale})")
# Generate the crop preview if crop coordinates exist
if crop:
preview_path = os.path.join(self.output_dir, f"{self.file_base}_f{start_fc}_CropPreview.mp4")
print(f"[DRY-RUN] Generating crop visualization video to {preview_path}...")
try:
self.processor.generate_crop_preview(self.input_path, metadata, crop, preview_path)
produced.append(preview_path)
except Exception as e:
print(f"[WARN] Failed to generate crop preview: {e}", file=sys.stderr)
return produced
try:
if process_type == "video":
self.processor.export_video(self.input_path, metadata, out_path, mode=mode, scale=scale)
elif process_type == "gif":
self.processor.export_gif(self.input_path, metadata, out_path, scale=scale)
else:
raise LoopProcessingError(f"Unsupported process type: {process_type}")
produced.append(out_path)
except LoopProcessingError:
raise
except Exception as e:
raise LoopProcessingError(f"Processing failed: {e}")
return produced
# =========================
# CLI
# =========================
def _parse_crop(crop_str: Optional[str]) -> Optional[Tuple[int, int, int, int]]:
if not crop_str:
return None
parts = crop_str.split(",")
if len(parts) != 4:
raise argparse.ArgumentTypeError("Crop must be x,y,w,h")
try:
return tuple(int(p) for p in parts)
except Exception:
raise argparse.ArgumentTypeError("Crop values must be integers")
def main(argv: Optional[List[str]] = None) -> int:
parser = argparse.ArgumentParser(
description="Loop Detection and Processing Utility (Stage 2: PreCure Pipeline)",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
# ========== REQUIRED ARGUMENTS ==========
parser.add_argument("input", help="Input video file")
# ========== DETECTION SETTINGS ==========
detect_group = parser.add_argument_group("Detection Settings")
detect_group.add_argument(
"-M",
"--method",
default="mse",
choices=LoopDetectorFactory.list_methods(),
help="Loop detection algorithm. "
"mse: Sliding Window MSE (optimal for animation on 2s/3s); "
"histogram: Histogram chi-squared (robust to color shifts); "
"perceptual: Perceptual hash (fast, scale-tolerant)",
)
detect_group.add_argument(
"--start",
required=True,
help="Start timecode (HH:MM:SS.mmm or frames)",
)
detect_group.add_argument(
"--end",
required=True,
help="End timecode (HH:MM:SS.mmm or frames)",
)
detect_group.add_argument(
"--crop",
help="Crop area (x,y,w,h)",
default=None,
)
detect_group.add_argument(
"--downscale",
type=str,
default="auto",
help="Downscale detection frames after crop. Examples: auto, 480, 480x270, 50%%",
)
# ========== OUTPUT AND PROCESSING SETTINGS ==========
process_group = parser.add_argument_group("Processing and Output Settings")
process_group.add_argument(
"-o",
"--output-dir",
default="output",
help="Directory to store CSV and exported files",
)
process_group.add_argument(
"--process",
choices=["video", "gif"],
help="Produce an exported clip",
)
process_group.add_argument(
"--mode",
choices=["normal", "prototype", "copy"],
default="normal",
help="Encoding mode for video exports",
)
process_group.add_argument(
"--scale",
help="Scaling for export (e.g., 640:-2 or 480:-1)",
default=None,
)
process_group.add_argument(
"--out-name",
help="Output filename (placed inside output-dir). "
"If omitted uses {file_base}_f{start_fc}_Loop-001.ext",
)
process_group.add_argument(
"--no-cache",
action="store_true",
help="Do not use cache; force re-detection",
)
process_group.add_argument(
"--dry-run",
action="store_true",
help="Do not export, only show actions",
)
# ========== ADVANCED SETTINGS ==========
advanced_group = parser.add_argument_group("Advanced Settings")
advanced_group.add_argument(
"--max-sample-frames",
type=int,
default=1000,
help="Max frames to sample during detection (auto-downsample if range larger)",
)
advanced_group.add_argument(
"--ref-block-size",
type=int,
default=5,
help="Reference block size (frames from start used to match candidates)",
)
args = parser.parse_args(argv)
try:
crop = _parse_crop(args.crop) if args.crop else None
wrapper = LoopWrapper(
args.input,
args.output_dir,
detection_method=args.method,
max_sample_frames=args.max_sample_frames,
ref_block_size=args.ref_block_size,
)
print(f"[INFO] Initialized for video: {args.input}")
print(f"[INFO] Using detection method: {args.method}")
metadata = wrapper.detect_loop(
args.start,
args.end,
crop=crop,
downscale_arg=args.downscale,
use_cache=not args.no_cache,
)
print(
f"[INFO] Detected loop: {metadata['start_tc']} -> {metadata['end_tc']} "
f"(frames {metadata['start_fc']}-{metadata['end_fc']}) "
f"quality={metadata.get('quality', 0.0):.3f} "
f"similarity={metadata.get('similarity', 0.0):.2f}"
)
if crop:
print(f"[INFO] Crop applied: x={crop[0]}, y={crop[1]}, w={crop[2]}, h={crop[3]}")
produced = wrapper.process(
metadata,
args.process,
args.mode,
args.scale,
args.out_name,
dry_run=args.dry_run,
crop=crop,
)
if produced:
for p in produced:
print(f"[INFO] Produced: {p}")
return 0
except LoopInputError as e:
print(f"[ERROR] Input error: {e}", file=sys.stderr)
return 1
except LoopDetectionError as e:
print(f"[ERROR] Detection error: {e}", file=sys.stderr)
return 2
except LoopProcessingError as e:
print(f"[ERROR] Processing error: {e}", file=sys.stderr)
return 3
except KeyboardInterrupt:
print("\n[WARN] Interrupted by user", file=sys.stderr)
return 130
except Exception as e:
print(f"[ERROR] Unexpected error: {e}", file=sys.stderr)
import traceback
traceback.print_exc(file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())

PreCure Production Pipeline: Scene & Loop Extraction Guide

This document outlines the operational workflow for isolating, auditing, and extracting high-quality video scenes and repeating animation loops from animated media (specifically episodic weekly anime formats like Pretty Cure). This guide is designed for human operators, open-source contributors, and automated AI agents to maintain data integrity across distributed execution environments.


1. High-Level Architectural Pipeline

The pipeline uses a two-stage approach. First, it processes an entire episode to isolate structural boundaries (Scenes). Second, it down-samples individual scenes to pinpoint micro-timings where frame-accurate repetitions occur (Loops).

Both workflows enforce a Proxy-Production separation: computationally expensive computer vision operations and manual human audits happen using highly compressed, lightweight previews, while final asset deployment runs headlessly on master source files.

[Master Video File]
        │
        ├─► [Stage 1: Scene Detection] ──► Low-Res Proxies ──► Human CSV Audit ──► High-Quality Scenes
        │                                                                               │
        └───────────────────────────────────────────────────────────────────────────────┤
                                                                                        ▼
                                                                           [Stage 2: Loop Extraction]
                                                                                        │
                                           High-Quality Assets ◄── Human Choice ◄── Top-N Loop Candidates


2. Stage 1: Scenes Detection Workflow

Anime episodes contain predictable but variable structures (recaps, dynamic openings, main content, endings, and next-episode previews). Stage 1 segments the video file into a trackable sequence.

Step-by-Step Execution

  1. Initial Detection Pass: Run scene_cutter.py with an adaptive or content threshold to determine raw visual cuts.
  2. Data Versioning: The output data is compiled into a standardized CSV file ({file_base}_scenes_{detection_method}.csv). Crucial: This initial CSV must immediately be checked into a local Git tracking branch or duplicated as a backup.
  3. Proxy Rendering: Generate low-quality, scaled-down preview clips using fast encoding profiles (e.g., -e prototype with a high Constant Rate Factor like CRF 32). This saves CPU/GPU cycles and disk space. To ensure these files don't collide, filenames must explicitly map their index: {file_base}_Clip-{num}.mp4
  4. Human Visual Audit: The operator reviews the low-quality previews alongside the CSV. The operator modifies the CSV file directly to:
  • Delete rows representing low-priority repetitive segments (Openings and Endings).
  • Combine overly brief cuts by merging their timestamps into a single, cohesive scene row.
  1. Selective Re-rendering: If rows are altered or deleted, the script is run sequentially using selection flags (-s) to update only the affected scene previews, checking for visual continuity.
  2. Master Asset Production: Once the audited CSV contains zero errors, it is pushed to background compute nodes (other PCs or high-performance runners) to extract scenes from the master file at absolute highest quality (-e copy or lossless re-encode).
  3. Archival: The final corrected CSV is preserved inside a repository for future structural reuse.

3. Stage 2: Loops Detection Workflow

The loop extraction pipeline depends entirely on filtered timestamps generated during Stage 1. It isolates looping subject animations (such as character transformations, magic spells, floating idles, or repeating actions) to produce source material for transparent Discord stickers (APNG/WebP) and web graphics (Tenor/GIPHY).

Algorithmic Logic: Sliding Window MSE

The default detection engine uses a Sliding Window Mean Squared Error (MSE) mechanism:

  • Targeting: A sub-segment range (--start and --end) is isolated from a known scene.
  • Cropping: Optional --crop boundaries isolate the subject coordinates (x, y, w, h) to eliminate moving backgrounds or panning cameras that skew mathematical comparison.
  • Grayscale Downscaling: Frames are downscaled (e.g., 640px wide) and stripped of color data to optimize computation matrix sizes.
  • Evaluation: A tiny sequence at the front (the ref_block_size) is used as a benchmark template. The script slides this template across the later sections of the clip, computing the difference in pixel intensities. The lowest error points represent the smoothest looping boundaries.

Step-by-Step Execution

  1. Bound Extraction: Identify target scene frames from the Stage 1 CSV. Thin down the operational windows closely to where the loop is likely to occur to minimize cache size.
  2. Loop Candidate Generation: Run loop_detector.py to evaluate frame differentials. The script reads and writes cache metadata dynamically appended with the unique starting frame boundaries ({file_base}_f{start_fc}_loops_{detection_method}.csv) to guarantee sequential runs on the same episode do not overwrite existing data.
  3. Multi-Candidate Preview Export: Generate low-res preview videos. To ensure these files don't collide, filenames must explicitly map their index: {file_base}_f{start_fc}_Loop-{num}.mp4
  4. Human Selection & Verification: The human auditor evaluates candidate frames. Because hand-drawn animation often holds matching drawings across multiple frames (animating on 2s or 3s), the human confirms the absolute cleanest loop boundaries to eliminate visual hitches.
  5. Asset Reproduction & Mastering:
  • Path A (Web GIFs): The exact frame boundaries from the verified CSV are used to build high-quality GIFs or video loops with an enforced minimum duration of $\ge 1.0$ second via container-level looping (-stream_loop).
  • Path B (Transparent Stickers): The bounding coordinates (--crop) and timecodes are passed downstream to background separation layers for transparency rotoscoping, preparing 320x320 pixel WebP/APNG assets.

4. Pipeline Vulnerabilities & Protocols

When directing automation layers or collaborating with other developers, strictly adhere to the following guardrails:

  • Strict CSV Parsing Rules: Manual editing of CSV strings runs the risk of introducing syntax exceptions. Ensure that automated parsers implement structural validation catches before passing data to split_video_ffmpeg.
  • Dynamic Structural Variance: Do not use hardcoded time offsets to drop openings/recaps across a full season. Pretty Cure and similar weekly shows alter internal composition for structural emphasis (e.g., recaps, extended movie promotions, or omitted openings). Always verify structures per-episode via Stage 1 previews.
  • Coordinate Data Integrity: The --crop specifications determined during loop evaluation must be treated as critical metadata and written directly into the production CSV rows; downstream transparency utilities require these bounds to process transparency masks accurately.
@gphg

gphg commented Jun 30, 2026

Copy link
Copy Markdown
Author

PySceneDetect wrapper: https://gist.github.com/gphg/117f2403f8bb91b6e00e11db97b6ffb7

  • Detect the scene and produces CSV data file easier
  • Known as scene_cutter.py

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment