Cmd foo

Find / Transform / Copy

Search for line in files

find ./ -type f -print0 | xargs -0 grep -l "Oh Hai"

Move matching files

# Find all files that start with "2016" and move them into a folder named "2016"
find ./ -maxdepth 1 -type f -iname '2016*' | xargs -I '{}' mv {} 2016/

More Examples

• Xargs With Examples

Search Django

BASE_PATH="~/<project>/env/lib/python2.7/site-packages"
find "$BASE_PATH/django" -type f -not -path '*/\.*' -name '*.py' -print0 | xargs -0 grep -C 3 -A 2 "Search String"

Merge multiple text files

#!/bin/bash
PATH=/bin:/usr/bin
export PATH
SRC_PATH="/home/tom/Desktop/notes/"
DEST_FILE="/home/tom/Desktop/scratch"
if [ -d "$SRC_PATH" ]; then
    cd $SRC_PATH
    echo ${PWD}
    find ./ -maxdepth 1 -type f -iname "*.txt" -print0 | while IFS= read -r -d $'\0' line;
    do
        echo -e "#$line\n" >> $DEST_FILE
        cat "$line" >> $DEST_FILE
        echo -e "\n\n\n" >> $DEST_FILE
    done
fi

Remove lines from tsv

Note entirely sure what this is doing

gzip -cd filename.tsv.gz | cut -f2,3 | sort -u | cut -f1 | uniq -c | grep -v " 1 " | cut -c9-

Backup Dropbox

rsync -vrut -n \
    --exclude="*.gpg" \
    --exclude="**/.git*" \
    --exclude="work" \
    --exclude="travel" \
    --exclude="wallpapers" \
    /home/tom/Dropbox/ /home/tom/dropbox_notes

Bulk Rename using cut

for f in *.mp3; do n=$(echo "$f" | cut -d' ' -f1,3-);mv "$f" "$n";done

File transformed via cut and saved to the variable n. The cut command could be swapped sed or any other manipulation program.

Newlines in bash ban be swapped out for ; symbols

Remove stale .pyc files

find ./ -name \*.pyc -ls
find ./ -name \*.pyc -exec rm {} \;
find ./ -name \*.pyc -ls

Decrypt

for f in *.gpg; do n=$(echo "$f" | cut -d'.' -f1,2);gpg -o "$n" -d "$f";done

Bulk converting epub to pdf

#!/bin/bash
for file in *.epub
do
    filename=$(basename -- "$file")
    extension="${filename##*.}"
    filename="${filename%.*}"
    echo "Procesing $filename..."
    pandoc \
        -V geometry:a5paper \
        -V geometry:margin=1.5cm \
        -V mainfont="DejaVu Serif" \
        -V monofont="DejaVu Sans Mono" \
        --pdf-engine=xelatex \
        -o "pdf/$filename.pdf" "$filename.epub"
done

Some good tips via https://learnbyexample.github.io/customizing-pandoc/ Still want to refine and tweak this a little more.

Put all files into folders of the same name

#!/bin/bash
find ./ -maxdepth 1 -type f -print0 | while IFS= read -r -d $'\0' line;
    do
        if [[ "$line" != *"cleaner.sh"* ]]; then
        filename=$(basename -- "$line")
        name=${filename%.*}
        if [ ! -d "$name" ]; then
                mkdir "$name"
            fi
        mv "$filename" "$name"
        fi
    done

Put all video files into folders of the same name

#!/bin/bash
find ./ -maxdepth 1 -type f -print0 | while IFS= read -r -d $'\0' line;
    do
        if [[ "$line" =~ ^.+\.(avi|mkv|iso|mp4) ]]; then
            filename=$(basename -- "$line")
            name=${filename%.*}
            echo "$name"
            if [ ! -d "$name" ]; then
                 mkdir "$name"
            fi
            mv "$filename" "$name"
        fi
    done

Redirection

via https://askubuntu.com/a/625230

1. Redirect stdout to one file and stderr to another file:

command > out 2>error

1. Redirect stdout to a file (>out), and then redirect stderr to stdout (2>&1):

command >out 2>&1

1. Redirect both to a file (bash & zsh only)

command &> out

Make MKV automation

Solution via https://forum.makemkv.com/forum/memberlist.php?mode=viewprofile&u=21164

# set main variables
TITLE="Movie Name (YEAR)"
INPUT="/home/user/${TITLE} UHD.mkv"
OUTPUT="/home/user/${TITLE} 4K ffmpeg 2 Audio.mkv"

# read hdr variables from source file
ffprobe \
  -loglevel quiet \
  -read_intervals "%+#2" \
  -select_streams v:0 \
  -show_entries side_data "${TITLE} UHD.mkv" | egrep -m 1 -A 10 'Mastering display metadata' | grep -v 'Mastering display metadata' >/tmp/variables.txt
ffprobe \
  -loglevel quiet \
  -read_intervals "%+#2" \
  -select_streams v:0 \
  -show_entries side_data "${TITLE} UHD.mkv" | egrep -m 1 -A 2 'Content light level metadata' | grep -v 'Content light level metadata' >>/tmp/variables.txt
source /tmp/variables.txt
cat /tmp/variables.txt

# ffmpeg transcode
ffmpeg \
  -y \
  -i "${INPUT}" \
  -t 00:05:00 \
  -hide_banner \
  -stats \
  -loglevel quiet \
  -analyzeduration 100M \
  -probesize 100M \
  -max_muxing_queue_size 9999 \
  -map 0:v:0 \
  -map 0:a:0 \
  -map 0:a:0 \
  -vf "crop=3840:1600:0:280" \
  -pix_fmt yuv420p10le \
  -vcodec libx265 \
  -preset slow \
  -x265-params "keyint=60:bframes=3:vbv-bufsize=75000:vbv-maxrate=75000:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:master-display='G("$green_x","$green_y")B("$blue_x","$blue_y")R("$red_x","$red_y")WP("$white_point_x","$white_point_y")L("$max_luminance","$min_luminance")':max-cll='"$max_content","$max_average"'" \
  -crf 24 \
  -force_key_frames "expr:gte(t,n_forced*2)" \
  -c:a:0 copy \
  -c:a:1 dca \
  -ab 1509k \
  -map_metadata -1 \
  -metadata title="${TITLE}" \
  -metadata:s:a:0 language=eng \
  -metadata:s:a:1 language=eng \
  -map_chapters -1 \
  -sn \
  -strict \
  -2 "${OUTPUT}"