ShellScriptingCookbook

Notes about foo in bash and zsh. For information on external unix commands to call from the shell, refer to my UnixCookbook.

Pointers

• UnixTime - date command fu

Quoting

• quoting double quotes inside double quoted strings - \"
• quoting a literal single quote inside single quoted string
  • '\'' = perl -le 'print "'\'' is a single quote"'
  • \047 = perl -le 'print "\047 is a single quote"'

grep

    # grep text files and skip binaries
    grep <string> `file * | egrep 'script|text' | awk -F: '{print $1}'`

find

    find ./ -name "A*" -name "*Page" ! -name "Apache*"

find large files

    find / -xdev -size +1024 -exec ls -al {} \; | sort -r -k 5

finding files using perl regexp

    # this is pretty useless - argument list too long...
    perl -nle 'print "$ARGV:$_" if m/^[A-C]/' `find . -type f`
    vi `perl -nle 'print $ARGV if m/^[A-C]/' `find . -type f``

File and directory names

    # get file name without directory
    basename /path/to/file.txt

    # also remove the extension from the file
    basename /path/to/file.txt .txt

Who's on /path/to/first

    # determine which users are using specified file
    fuser -u /path/to/first
    lsof | grep /path/to/first

Randomness

    more /dev/random | head -10 | openssl md5

touch

• see also UnixTime

    touch -t ccyymmddhhMMSS filename

    # touch a file with the time one hour ago - freebsd date
    touch -t $(date -v '-1H' '+%Y%m%d%H%M') ~/testme

calculator

     bc

line numbers

    cat -n file > file.num

    nl file.c > file.c.num

    # numbers blank lines
    nl -ba file.c >file.c.all_num

Wrap lines at 80 characters

    fold filename

File reversal

    cat <file-name> | awk '{ line[NR] = $0;} END {for(i=NR;i>1;i--) print line[i] }'

file similarity

    diff $file1 $file2 | wc -l

diffs

    ediff

    # color
    diff -u file1 file2 > output; vim $output

Binary file diffs

    # dump files into text octal and ascii and diff the text files
    od -c binfile1 > binfile1.txt
    od -c binfile2 > binfile2.txt
    diff binfile1.txt binfile2.txt

search and replace with tr

    tr -d '^M' <inputfile >outputfile
    replace multiple spaces with a single space: tr -s ' '

* sorted wordlist

<example>
    | tr -c '[:alnum:]' '[\n*]' | sort -iu | grep -v '^[0-9]*$'

dd

    # create a blank file
    dd if=/dev/zero count=5000000 of=testfile

    # blank out a disk
    dd if=/dev/urandom of=/dev/disk
    dd if=/dev/zero of=/dev/disk
    dd if=/dev/urandom of=/dev/disk

cpio

    cd /path/to/src/dir; find . | cpio -pdumv /path/to/dest/dir

awk

    # ping hosts in /etc/hosts
    awk '/^[0-9]/{print $1}' /etc/hosts |xargs -l ping -c 1 $1
    awk '/^[^#]/ {system("ping -c 1 "$1)}' < /etc/hosts

man

    # specify your man path
    man -M manpath manpage

    # convert man page to text - get rid of formatting characters
    man manpage | col -b > manpage.txt

file consistency check

    # check that a file (e.g. csv) has consistent number of fields on each line
    nawk -F "delimiter" '{print NF}' file_name|sort -u|more

Handling failure

    # if the exit status is non-zero, tell the user it failed
    if [ "$?" -gt 0 ]; then echo FAILED; fi

Beep Beep

    # echo a beep
    echo $'\a'

    # ctrl-v (next char is literal), ctrl-g
    echo ^V^G

remove blank lines

    # remove lines that are blank.  doesn't remove lines with spaces/tabs.
    sed -e '/^$/d' filetoread >filetowrite
    cat file1 | grep -v '^$' >file2
    perl -ne 'print unless /^$/' < infile

    # use space and tab character in []
    sed -e '/^[     ]*$/d' InputFile >OutputFile

Reading from a file

    while read file; do rm -rfv /path/to/jail$file; done < jail_remove.txt

Wrappers

    #!/bin/sh
    umask 007
    /path/to/wrapped/command "$@"

Insert a line

    string="hello"
    ed << EOF
    e any_file
    1i
    ${string}

Path

    echo $PATH | tr ":" "\n"

String Operators

    ${varname:-word} = return varname if exists and not null, else
                       return word.  useful to return a default value
                       if variable undefined.

    ${varname:=word} = return varname if exists and not null, else set
                       to word and return value.  useful to set
                       variable to a default value.

    ${varname:?word} = if varname exists and isn't null, return it's
                       value, else print varname and message, and
                       abort.  useful for catching undefined
                       variables.

    ${varname:+word} = if varname exists and isn't null, return wrod,
                       else return null.  useful for testing existence
                       of a variable.

    ${varname:offset} = return substring of varname starting at
                        offset.

    ${varname:offset:length} = return substring of varname starting at
                               offset up to length characters.

Patterns and Pattern Matching

    ${variable#pattern} = delete shortest match from the beginning
    ${variable##pattern} = delete longest match from the beginning
    ${variable%pattern} = delete shortest match from the end
    ${variable%%pattern} = delete longest match from the end
    ${variable/pattern/string} = replace longest pattern match with string
    ${variable//pattern/string} = replace all (longest) pattern matches with string

Test Operators

    -d file = file exists and is a directory
    -e file = file exists
    -f file = file exists and is a regular file

    -r file = read perms on file
    -s file = file exists and is not empty
    -w file = file is writable
    -x file = file is executable

    -O file = you own file
    -G file = gid on file matches one of yours

    file1 -nt file2 = file1 is newer than file2
    file1 -ot file2 = file1 is older than file2

    $x -lt $y = x < y
    $x -le $y = x <= y
    $x -eq $y = x = y
    $x -ge $y = x >= y
    $x -tg $y = x > y
    $x -ne $y = x != y

Case statements

case expression in
  pattern1 )
    statements ;;
  pattern2 )
    statements ;;
esac

Select

select name [in list]
do
  statements that can use $name
done

while and until

    while condition
    do
      statements
    done

    until command; do
      statements
    done

    # bash - same as "while true"
    while :
    do
      sleep x
    done

getopts

while getopts ":ab:c" opt; do
  case $opt in
    a  ) process option -a ;;
    b  ) process option -b
         $OPTARG is the option's argument ;;
    c  ) process option -c ;;
    \? ) echo 'usage: alice [-a] [-b arg] [-c] args...'
         exit 1
  esac
done
shift $(($OPTIND - 1))

# normal processing of arguments...

echo escape sequences

    \a = alert or ctrl-g (bell)
    \b = backspace or ctrl-h
    \c = omit final newline
    \E = escape character
    \f = formfeed or ctrl-l
    \n = newline or ctrl-j
    \r = return or ctrl-m
    \t = tabl or ctrl-i
    \v = vertical tab or ctrl-k
    \n = ascii char with octal value n
    \\ = single backslash

read

    $ read character1 character2
    alice duchess
    $ echo $character1
    alice
    $ echo $character2
    duchess

background jobs

    %N = job number N
    %string = job whose command begins with string
    %?string = job whose command contains string
    %+ = most recently invoked background job
    %% = most recently invoked background job
    %- = second most recently invoked background job

trap

    # Signal Info
    # ctrl-c = INT (interrupt)
    # ctrl-z = TSTP (terminal stop)
    # ctrl-\ = QUIT


    # usage
    trap cmd sig1 sig2

    # trap ctrl-c
    trap "echo 'You hit control-C!'" INT

    # remove a lock file on exit
    trap "{ rm -f $LOCKFILE ; exit 255; }" EXIT
    trap 'rm -f $TEMPFILE; exit 13' TERM INT

    # disable/ignore signhup
    trap "" HUP

    # reset handlers for INT and TERM signals
    trap - INT TERM

    # fake signal EXIT occurs when script exits
    trap 'echo exiting from the script' EXIT

    # fake signal DEBUG when shell executes a statement
    trap "echo badvar is badvar" DEBUG

Bash Debugging

• bash - tracing variables
  • DEBUG trapped after every command in a script

  declare -t VARIABLE=value trap 'echo "VARIABLE-TRACE> \$variable = \"$variable\""' DEBUG

sh

- sh -n scriptname checks for syntax errors without actually running the script

- sh -v scriptname echoes each command before executing it.

• the equivalent of inserting set -v or set -o verbose in the script.

- sh -nv scriptname gives a verbose syntax check.

- sh -x scriptname echoes the result each command in an abbreviated manner.

• This is the equivalent of inserting set -x or set -o xtrace in the script

• Inserting set -u or -set -o nounset= in the script gives an unbound variable error message at each attempt to use an undeclared variable.

- general

• EXIT trap to print key variable values

  trap 'echo Variable Listing --- a is $a b is $b' EXIT

Hash

    # Pseudo hash
    name=test
    test=asdf

    # lookup value of TEST (asdf) by name
    eval echo \${$NAME}

Pipe command to block

    # FD 4 becomes stdin too
    exec 4>&0

    xauth list "$DISPLAY" | sed -e 's/^/add /' | {

        # FD 3 becomes xauth output
        # FD 0 becomes stdin again
        # FD 4 is closed
        exec 3>&0 0>&4 4>&-

        exec su - "$CLIENTUSER" -c \
             "xauth -q <&3
              exec env DISPLAY='$DISPLAY' "'"$SHELL"'" -c '$*' 3>&-"

    }

Tar

# move a file heirarchy
tar -cf - -C srcdir . | tar xpf - -C destdir
# move a file heirarchy - otter
tar cf - . | (cd /somewhere && tar xvf -)


# same system - otter
gzip -dc xxx.tgz | tar xvf -

# across systems - otter
cat xxxx.tgz | ssh host -c '(cd /prod && (gzip -dc | tar xvf -))'