From a64b18f46c6eea77237a33a75c320182f0e249cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Mei=C3=9Fner?= <936176+t-nil@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:13:39 +0200 Subject: [PATCH] update video scripts --- video/encode_games.fish | 67 +++++++++++++++++++++++++++-------------- video/stats.fish | 31 ++++++++++++++++--- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/video/encode_games.fish b/video/encode_games.fish index b153573..786d13f 100755 --- a/video/encode_games.fish +++ b/video/encode_games.fish @@ -8,6 +8,12 @@ set helpers (dirname (realpath (status filename)))/../helpers set RULES encode_rules.conf +set _MINI_GOP 32 # actual value from svtav1 + + +# ----------------------------------------------------------- + + cd /run/media/ra1n/8tb_backup/Videos/Games @@ -61,7 +67,7 @@ end # --- Zone stuff --- -function get_frame_rate -a file +function get_fps -a file ffprobe -v error -select_streams v -of default=noprint_wrappers=1:nokey=1 -show_entries stream=r_frame_rate $file end @@ -101,29 +107,29 @@ function extract_zones_from_chapters -a file set -q _DEBUG && echo --- Chapter OGM \($file\) ---\n $chapters\n --- END --- >&2 [ "$chapters" = "" ] && return 2 - set frame_rate (get_frame_rate $file) + set fps (get_fps $file) set frame_count (get_frame_count $file) - set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): $frame_count =' $frame_count, '$frame_rate =' $frame_rate; set_color normal; end >&2 + set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): $frame_count =' $frame_count, '$fps =' $fps; set_color normal; end >&2 set chapters (string join \n $chapters | while read -L timestamp name set timestamp (string match -rg 'CHAPTER\d{2}=(.*)' $timestamp) set name (string match -rg 'CHAPTER\d{2}NAME=(.*)' $name) - echo (math floor\((hhmmss_to_secs $timestamp) \* $frame_rate\))\|$name + echo (math floor\((hhmmss_to_secs $timestamp) \* $fps\))\|$name end) - set -a chapters (math $frame_count)\| # \| dummy separator for splitting below + set -a chapters -1\| # \| dummy separator for splitting below set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): == intermediate chapters =='; string join -- \n $chapters; set_color normal; end >&2 set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): == final zones =='; set_color normal; end >&2 for i in (seq 1 (count $chapters[2..])) - set -l input (string split --max 1 \| $chapters[$i]) + set -l input (string split --max 1 -- \| $chapters[$i]) set -l start_frame $input[1] - set -l end_frame (string split --max 1 --fields 1 \| $chapters[(math $i +1)]) - set -l zone_marker (string match -rg '^_(.*)_$' $input[2]) + set -l end_frame (string split --max 1 --fields 1 -- \| $chapters[(math $i +1)]) + set -l zone_marker (string match -rg -- '^_(.*)_$' $input[2]) set -l encoder svt-av1 set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters():' \$input=$input, \$start_frame=$start_frame, \$end_frame=$end_frame, \$zone_marker=$zone_marker, \$encoder=$encoder; set_color normal; end >&2 - if [ "$end_frame" -le "$start_frame" ] + if [ "$end_frame" -le "$start_frame" ] && [ "$end_frame" -ne -1 ] # special case for last frame, which is -1 (stands for last movie frame in Av1an) set_color yellow; echo 'extract_zones_from_chapters(): zone' $zone_marker 'is invalid: (' $start_frame '>=' $end_frame ')'; set_color normal continue end >&2 @@ -134,18 +140,18 @@ function extract_zones_from_chapters -a file # start_frame end_frame encoder reset(opt) video_params switch $zone_marker case pause - set -f -- params reset --extra-split (math $frame_rate \* 120.3) --keyint 0 --crf 70 --preset 11 + set -f -- params reset --extra-split (math $fps \* 120) --keyint 0 --crf 70 --preset 11 --enable-variance-boost 0 case still - set -f -- params reset --extra-split (math floor\($frame_rate \* 45.3\) ) --keyint 0 --crf 63 --preset 8 --enable-dlf 2 --tune 0 + set -f -- params reset --min-scene-len (mini_gop_offset 28 $fps 32) --extra-split (mini_gop_offset 40 $fps 32) --keyint 0 --crf 61 --preset 8 --enable-variance-boost 0 --enable-dlf 2 --tune 0 case hq - set -f -- params reset --extra-split (math floor\($frame_rate \* 16 / 32\) \*32 +1 ) --keyint 0 --crf 28 --preset 5 --frame-luma-bias 35 --tune 0 --film-grain 8 --film-grain-denoise 0 --enable-variance-boost 1 --variance-boost-strength 3 --variance-octile 5 + set -f -- params reset --keyint 0 --crf 35 --preset 5 --frame-luma-bias 42 --tune 0 --film-grain 8 --film-grain-denoise 0 --enable-variance-boost 1 --variance-boost-strength 3 --variance-octile 5 case \* continue end set -l -- zone $start_frame $end_frame $encoder $params echo $zone - set -q _DEBUG && begin; set_color -di; echo 'extract_zones_from_chapters(): => ' $zone '($input =' $input, '$zone_marker =' $zone_marker\); set_color normal; end >&2 + begin; set_color -di; echo 'extract_zones_from_chapters(): => ' $zone '($input =' $input, '$zone_marker =' $zone_marker\); set_color normal; end >&2 end end @@ -169,7 +175,7 @@ while [ (count $filelist) -gt 0 ] echo -n (string pad $output --right --width $to_pad) string match --entire '[*]' $output >/dev/null || begin; set_color brblack; echo SKIPPING, not tagged; set_color normal; continue; end # file names without atleast one pair of [] brackets are not tagged and will be skipped get_length $f - set -l frame_rate (get_frame_rate $f) + set -l fps (get_fps $f) [ -e "$output" ] && continue set_color normal @@ -187,25 +193,41 @@ while [ (count $filelist) -gt 0 ] set G 0 # correct for the +1 on the chunker, so we dont accidentally emit two keyframes after one another set P 5 set CRF 51 + set GRAIN 20 + + string match -qe '[Psychonauts 2]' $f && set -l GRAIN 35 + + set -l fps (get_fps $f) - set frame_rate (get_frame_rate $f) + set -l zones (extract_zones_from_chapters $f) - set zones (extract_zones_from_chapters $f) + # we don't need to adapt the zones to the target fps, because Av1an does + # the zone splitting based on the "real" frames in the video. The video + # encoder, which sees the "target" frames, doesn't know anything about + # them anymore as it only sees one chunk and has zone settings applied + # to its whole process. + string match -rqi '\[other.*\]' $f && set -l target_fps 15 + string match -rqi '\[.*watch.*\]' $f && set -l target_fps 30 + string match -rqi '\[no_picture\]' $f && set -l target_fps 1 + set -q target_fps && begin; set_color -di; echo '=> $target_fps = '$target_fps; set_color normal; end >&2 + set -q _DEBUG && set fish_trace on set cmdline (string escape -- schedtool -B -e nice -n 19 \ av1an --verbose -n -i $f -o $output \ --log-file logs/$output --log-level debug --resume --workers $WORKERS (: --set-thread-affinity $N_THREADS "not effective _ set to same as --lp on svt") \ - --extra-split (math 16.3 \* $frame_rate) (: "previous: 60*30*1 == 30 sec chunks, should be 3 keyframes with `-g 600`") (: --split-method none "no scenechange as we follow SVT instructions to split at defined intervals") \ + --extra-split (mini_gop_offset 10 $fps $_MINI_GOP) --min-scene-len (mini_gop_offset 8 $fps $_MINI_GOP) (: "previous: 60*30*1 == 30 sec chunks, should be 3 keyframes with `-g 600`") \ + --sc-downscale-height 360 --sc-method standard --split-method none (: "no scenechange as we follow SVT instructions to split at defined intervals") \ --encoder svt-av1 --passes 1 (: "two-pass doesn't seem to work on SvtAv1Psy, maybe because av1an passes in --pass 3, or simply wrong documentation") \ - --video-params "--lp "(math "$N_THREADS")" --keyint $G --crf $CRF --tune 0 --preset $P --sharpness 2 --film-grain 20 --film-grain-denoise 0 --enable-variance-boost 1 --frame-luma-bias 35" \ - --ffmpeg '-vf scale=-1:1080:flags=lanczos' \ - --chunk-order random --concat mkvmerge (: "appearently the best") \ + --video-params "--lp "(math "$N_THREADS")" --keyint $G --crf $CRF --tune 0 --preset $P --sharpness 2 --film-grain $GRAIN --film-grain-denoise 0 --enable-variance-boost 1 --frame-luma-bias 35" \ + --ffmpeg '-vf scale=-1:\\\'min(ih,1080)\\\':flags=lanczos'(set -q target_fps && echo -- ',fps='$target_fps || echo '') \ + --chunk-order random --concat mkvmerge (: "appearently the best") (set -q target_fps && echo -- --ignore-frame-mismatch) \ --vmaf --vmaf-threads 2 (: "just guessing, we don't want to steal too much CPU from encoding and vmaf scales logarithmically or so")) \ ([ (count $zones) -gt 0 ] && begin set -l tempfile (mktemp --suffix .zones.txt) string join \n $zones >$tempfile string join -- \n --zones $tempfile end) + set -q _DEBUG && set -e fish_trace else set cmdline (string escape -- schedtool -D -e nice -n 19 ffmpeg -n -hide_banner -i $f -map 0 -c copy -c:v libsvtav1 -pix_fmt yuv420p10le -g 600 $params -f matroska) @@ -213,9 +235,8 @@ while [ (count $filelist) -gt 0 ] set -a cmdline (string escape -- $output) end - set -q DRY_RUN \ - && echo $cmdline \ - || eval $cmdline #&| tr \n \r + set_color -di blue; echo $cmdline; set_color normal + set -q DRY_RUN || eval $cmdline #&| tr \n \r if [ "$pipestatus[1]" != 0 ] cleanup $output diff --git a/video/stats.fish b/video/stats.fish index 7098de3..3dd6497 100755 --- a/video/stats.fish +++ b/video/stats.fish @@ -5,7 +5,8 @@ set is_numeric __t-nil_helper__is_numeric argparse 'sort-by=' 'm/megabytes' -- $argv -set $_flag_sort_by (set -q $_flag_sort_by && echo $_flag_sort_by || echo 3) +set -- _flag_sort_by (set -q _flag_sort_by && echo -- -k $_flag_sort_by || echo -- -k 1) + function __t-nil_helper_extract --argument-names key string join \n $argv[2..] | string replace --filter $key'=' '' @@ -13,8 +14,24 @@ end set extract __t-nil_helper_extract + +set term_width (tput cols) + +if functions -q @test + @test 'extract ffprobe data' (set input 'duration=01:05:22.033' \ + 'width=1920' \ + 'height=1080' + begin + $extract duration $input + $extract width $input + $extract height $input + end | string join ' ') = "01:05:22.033 1920 1080" + + exit 0 +end + while read f - echo -ne $f\t + set output $f set blah (ffprobe -v error -select_streams v -show_entries stream=width,height,r_frame_rate:format=duration,bit_rate -of default=noprint_wrappers=1 $f 2>/dev/null | string trim) set duration ($extract duration $blah) @@ -35,10 +52,14 @@ while read f set duration (math --scale 0 $duration / 60):(math --scale 0 $duration % 60 | string pad --char 0 --width 2) set -q _flag_megabytes && [ "$bit_rate" != "N/A" ] && set bit_rate (printf '%.2f' (math -s 2 $bit_rate / 1024^2))' Mb/s' - echo -ne (string join \t $duration $bit_rate $width"x"$height $fps $bits_per_pixel) + set -a output $duration $bit_rate $width"x"$height $fps $bits_per_pixel - echo + set output_width (math (string length --visible $output[2..] | string join +)) + set left_for_name (math $term_width - $output_width - 50) + [ $left_for_name -lt 3 ] && begin; echo Terminal too small ':(' only $left_for_name chars for title left; exit 3; end + set output[1] (string shorten -m $left_for_name $output[1]) + string join -- \t $output end | pv -l | column --separator=\t --output-separator=\t --table \ --table-columns 'FILE,DURATION,BIT RATE,RESOLUTION,FPS,BITS PER PIXEL' \ --table-right 'DURATION,BIT RATE,RESOLUTION,FPS,BITS PER PIXEL' \ -| begin; sed -u 1q; sort -nt\t -k$_flag_sort_by; end # sed trick ignores header for sorting +| fish -c 'begin; sed -u 1q; sort -nt\t '$_flag_sort_by'; end' # sed trick ignores header for sorting -- GitLab