From a64b18f46c6eea77237a33a75c320182f0e249cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Florian=20Mei=C3=9Fner?=
 <936176+t-nil@users.noreply.github.com>
Date: Fri, 18 Oct 2024 11:13:39 +0200
Subject: [PATCH] update video scripts

---
 video/encode_games.fish | 67 +++++++++++++++++++++++++++--------------
 video/stats.fish        | 31 ++++++++++++++++---
 2 files changed, 70 insertions(+), 28 deletions(-)

diff --git a/video/encode_games.fish b/video/encode_games.fish
index b153573..786d13f 100755
--- a/video/encode_games.fish
+++ b/video/encode_games.fish
@@ -8,6 +8,12 @@ set helpers (dirname (realpath (status filename)))/../helpers
 set RULES encode_rules.conf
 
 
+set _MINI_GOP 32 # actual value from svtav1
+
+
+# -----------------------------------------------------------
+
+
 cd /run/media/ra1n/8tb_backup/Videos/Games
 
 
@@ -61,7 +67,7 @@ end
 
 # --- Zone stuff ---
 
-function get_frame_rate -a file
+function get_fps -a file
     ffprobe -v error -select_streams v -of default=noprint_wrappers=1:nokey=1 -show_entries stream=r_frame_rate $file
 end
 
@@ -101,29 +107,29 @@ function extract_zones_from_chapters -a file
     set -q _DEBUG && echo --- Chapter OGM \($file\) ---\n $chapters\n --- END --- >&2
     [ "$chapters" = "" ] && return 2
     
-    set frame_rate (get_frame_rate $file)
+    set fps (get_fps $file)
     set frame_count (get_frame_count $file)
-    set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): $frame_count =' $frame_count, '$frame_rate =' $frame_rate; set_color normal; end >&2
+    set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): $frame_count =' $frame_count, '$fps =' $fps; set_color normal; end >&2
     
     set chapters (string join \n $chapters | while read -L timestamp name
         set timestamp (string match -rg 'CHAPTER\d{2}=(.*)' $timestamp)
         set name (string match -rg 'CHAPTER\d{2}NAME=(.*)' $name)
-        echo (math floor\((hhmmss_to_secs $timestamp) \* $frame_rate\))\|$name
+        echo (math floor\((hhmmss_to_secs $timestamp) \* $fps\))\|$name
     end)
-    set -a chapters (math $frame_count)\| # \| dummy separator for splitting below
+    set -a chapters -1\| # \| dummy separator for splitting below
     set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): == intermediate chapters =='; string join -- \n $chapters; set_color normal; end >&2
     set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): ==      final zones      =='; set_color normal; end >&2
     
     for i in (seq 1 (count $chapters[2..]))
-        set -l input (string split --max 1 \| $chapters[$i])
+        set -l input (string split --max 1 -- \| $chapters[$i])
         set -l start_frame $input[1]
-        set -l end_frame (string split --max 1 --fields 1 \| $chapters[(math $i +1)])
-        set -l zone_marker (string match -rg '^_(.*)_$' $input[2])
+        set -l end_frame (string split --max 1 --fields 1 -- \| $chapters[(math $i +1)])
+        set -l zone_marker (string match -rg -- '^_(.*)_$' $input[2])
         set -l encoder svt-av1
         
         set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters():' \$input=$input, \$start_frame=$start_frame, \$end_frame=$end_frame, \$zone_marker=$zone_marker, \$encoder=$encoder; set_color normal; end >&2
 
-        if [ "$end_frame" -le "$start_frame" ]
+        if [ "$end_frame" -le "$start_frame" ] && [ "$end_frame" -ne -1 ] # special case for last frame, which is -1 (stands for last movie frame in Av1an)
             set_color yellow; echo 'extract_zones_from_chapters(): zone' $zone_marker 'is invalid: (' $start_frame '>=' $end_frame ')'; set_color normal
             continue
         end >&2
@@ -134,18 +140,18 @@ function extract_zones_from_chapters -a file
         # start_frame end_frame encoder reset(opt) video_params
         switch $zone_marker
             case pause
-                set -f -- params reset --extra-split (math $frame_rate \* 120.3) --keyint 0 --crf 70 --preset 11
+                set -f -- params reset --extra-split (math $fps \* 120) --keyint 0 --crf 70 --preset 11 --enable-variance-boost 0
             case still
-                set -f -- params reset --extra-split (math floor\($frame_rate \* 45.3\) ) --keyint 0 --crf 63 --preset 8 --enable-dlf 2 --tune 0
+                set -f -- params reset --min-scene-len (mini_gop_offset 28 $fps 32) --extra-split (mini_gop_offset 40 $fps 32) --keyint 0 --crf 61 --preset 8 --enable-variance-boost 0 --enable-dlf 2 --tune 0
             case hq
-                set -f -- params reset --extra-split (math floor\($frame_rate \* 16 / 32\) \*32 +1 ) --keyint 0 --crf 28 --preset 5 --frame-luma-bias 35 --tune 0 --film-grain 8 --film-grain-denoise 0 --enable-variance-boost 1 --variance-boost-strength 3 --variance-octile 5
+                set -f -- params reset --keyint 0 --crf 35 --preset 5 --frame-luma-bias 42 --tune 0 --film-grain 8 --film-grain-denoise 0 --enable-variance-boost 1 --variance-boost-strength 3 --variance-octile 5
             case \*
                 continue
         end
         set -l -- zone $start_frame $end_frame $encoder $params
         echo $zone
 
-        set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters():  => ' $zone '($input =' $input, '$zone_marker =' $zone_marker\); set_color normal; end >&2
+        begin; set_color -di; echo 'extract_zones_from_chapters():  => ' $zone '($input =' $input, '$zone_marker =' $zone_marker\); set_color normal; end >&2
     end
 end
 
@@ -169,7 +175,7 @@ while [ (count $filelist) -gt 0 ]
     echo -n (string pad $output --right --width $to_pad)
     string match --entire '[*]' $output >/dev/null || begin; set_color brblack; echo SKIPPING, not tagged; set_color normal; continue; end     # file names without atleast one pair of [] brackets are not tagged and will be skipped
     get_length $f
-    set -l frame_rate (get_frame_rate $f)
+    set -l fps (get_fps $f)
     [ -e "$output" ] && continue
     set_color normal
 
@@ -187,25 +193,41 @@ while [ (count $filelist) -gt 0 ]
         set G 0 # correct for the +1 on the chunker, so we dont accidentally emit two keyframes after one another
         set P 5
         set CRF 51
+        set GRAIN 20
+
+        string match -qe '[Psychonauts 2]' $f && set -l GRAIN 35
+
+        set -l fps (get_fps $f)
 
-        set frame_rate (get_frame_rate $f)
+        set -l zones (extract_zones_from_chapters $f)
 
-        set zones (extract_zones_from_chapters $f)
+        # we don't need to adapt the zones to the target fps, because Av1an does
+        # the zone splitting based on the "real" frames in the video. The video
+        # encoder, which sees the "target" frames, doesn't know anything about
+        # them anymore as it only sees one chunk and has zone settings applied
+        # to its whole process.
+        string match -rqi '\[other.*\]' $f && set -l target_fps 15
+        string match -rqi '\[.*watch.*\]' $f && set -l target_fps 30
+        string match -rqi '\[no_picture\]' $f && set -l target_fps 1
+        set -q target_fps && begin; set_color -di; echo '=> $target_fps = '$target_fps; set_color normal; end >&2
 
+        set -q _DEBUG && set fish_trace on
         set cmdline (string escape -- schedtool -B -e nice -n 19 \
             av1an --verbose -n -i $f -o $output \
                 --log-file logs/$output --log-level debug --resume --workers $WORKERS (: --set-thread-affinity $N_THREADS "not effective _ set to same as --lp on svt") \
-                --extra-split (math 16.3 \* $frame_rate) (: "previous: 60*30*1 == 30 sec chunks, should be 3 keyframes with `-g 600`") (: --split-method none "no scenechange as we follow SVT instructions to split at defined intervals") \
+                --extra-split (mini_gop_offset 10 $fps $_MINI_GOP) --min-scene-len (mini_gop_offset 8 $fps $_MINI_GOP) (: "previous: 60*30*1 == 30 sec chunks, should be 3 keyframes with `-g 600`") \
+                --sc-downscale-height 360 --sc-method standard --split-method none (: "no scenechange as we follow SVT instructions to split at defined intervals") \
                 --encoder svt-av1 --passes 1 (: "two-pass doesn't seem to work on SvtAv1Psy, maybe because av1an passes in --pass 3, or simply wrong documentation") \
-                --video-params "--lp "(math "$N_THREADS")" --keyint $G --crf $CRF --tune 0 --preset $P --sharpness 2 --film-grain 20 --film-grain-denoise 0 --enable-variance-boost 1 --frame-luma-bias 35" \
-                --ffmpeg '-vf scale=-1:1080:flags=lanczos' \
-                --chunk-order random --concat mkvmerge (: "appearently the best") \
+                --video-params "--lp "(math "$N_THREADS")" --keyint $G --crf $CRF --tune 0 --preset $P --sharpness 2 --film-grain $GRAIN --film-grain-denoise 0 --enable-variance-boost 1 --frame-luma-bias 35" \
+                --ffmpeg '-vf scale=-1:\\\'min(ih,1080)\\\':flags=lanczos'(set -q target_fps && echo -- ',fps='$target_fps || echo '') \
+                --chunk-order random --concat mkvmerge (: "appearently the best") (set -q target_fps && echo -- --ignore-frame-mismatch) \
                 --vmaf --vmaf-threads 2 (: "just guessing, we don't want to steal too much CPU from encoding and vmaf scales logarithmically or so")) \
                 ([ (count $zones) -gt 0 ] && begin
                     set -l tempfile (mktemp --suffix .zones.txt)
                     string join \n $zones >$tempfile
                     string join -- \n --zones $tempfile
                 end)
+        set -q _DEBUG && set -e fish_trace
 
     else
         set cmdline (string escape -- schedtool -D -e nice -n 19 ffmpeg -n -hide_banner -i $f -map 0 -c copy -c:v libsvtav1 -pix_fmt yuv420p10le -g 600 $params -f matroska)
@@ -213,9 +235,8 @@ while [ (count $filelist) -gt 0 ]
         set -a cmdline (string escape -- $output)
     end
 
-    set -q DRY_RUN \
-      && echo $cmdline \
-      || eval $cmdline #&| tr \n \r
+      set_color -di blue; echo $cmdline; set_color normal
+    set -q DRY_RUN || eval $cmdline #&| tr \n \r
 
     if [ "$pipestatus[1]" != 0 ]
         cleanup $output
diff --git a/video/stats.fish b/video/stats.fish
index 7098de3..3dd6497 100755
--- a/video/stats.fish
+++ b/video/stats.fish
@@ -5,7 +5,8 @@
 set is_numeric __t-nil_helper__is_numeric
 
 argparse 'sort-by=' 'm/megabytes' -- $argv
-set $_flag_sort_by (set -q $_flag_sort_by && echo $_flag_sort_by || echo 3)
+set -- _flag_sort_by (set -q _flag_sort_by && echo -- -k $_flag_sort_by || echo -- -k 1)
+
 
 function __t-nil_helper_extract --argument-names key
   string join \n $argv[2..] | string replace --filter $key'=' ''
@@ -13,8 +14,24 @@ end
 set extract __t-nil_helper_extract
 
 
+
+set term_width (tput cols)
+
+if functions -q @test
+    @test 'extract ffprobe data' (set input 'duration=01:05:22.033' \
+                                            'width=1920' \
+                                            'height=1080'
+                                  begin
+                                      $extract duration $input
+                                      $extract width $input
+                                      $extract height $input
+                                  end | string join ' ') = "01:05:22.033 1920 1080"
+
+    exit 0
+end
+
 while read f
-     echo -ne $f\t
+     set output $f
 
      set blah (ffprobe -v error -select_streams v -show_entries stream=width,height,r_frame_rate:format=duration,bit_rate -of default=noprint_wrappers=1 $f 2>/dev/null | string trim)
      set duration ($extract duration $blah)
@@ -35,10 +52,14 @@ while read f
      set duration (math --scale 0 $duration / 60):(math --scale 0 $duration % 60 | string pad --char 0 --width 2)
      set -q _flag_megabytes && [ "$bit_rate" != "N/A" ] && set bit_rate (printf '%.2f' (math -s 2 $bit_rate / 1024^2))' Mb/s'
 
-     echo -ne (string join \t $duration $bit_rate $width"x"$height $fps $bits_per_pixel)
+     set -a output $duration $bit_rate $width"x"$height $fps $bits_per_pixel
 
-     echo
+     set output_width (math (string length --visible $output[2..] | string join +))
+     set left_for_name (math $term_width - $output_width - 50)
+     [ $left_for_name -lt 3 ] && begin; echo Terminal too small ':(' only $left_for_name chars for title left; exit 3; end
+     set output[1] (string shorten -m $left_for_name $output[1])
+     string join -- \t $output
 end | pv -l | column --separator=\t --output-separator=\t --table \
                   --table-columns 'FILE,DURATION,BIT RATE,RESOLUTION,FPS,BITS PER PIXEL' \
                   --table-right 'DURATION,BIT RATE,RESOLUTION,FPS,BITS PER PIXEL' \
-| begin; sed -u 1q; sort -nt\t -k$_flag_sort_by; end  # sed trick ignores header for sorting
+| fish -c 'begin; sed -u 1q; sort -nt\t '$_flag_sort_by'; end'  # sed trick ignores header for sorting
-- 
GitLab