Subversion Repositories vgwhois

#!/bin/bash

#
#  VGWhoIs (ViaThinkSoft Global WhoIs, a fork of generic Whois / gwhois)
#  Maintenance / Developer utilities
#
#  (c) 2012-2019 by Daniel Marschall, ViaThinkSoft <info@daniel-marschall.de>
#
#  License: https://www.gnu.org/licenses/gpl-2.0.html (GPL version 2)
#

# TODO: use >&2

# TODO: anzeigen von wann der whois output ist... "last checked" -> "last activity" (da ein user-mode-batch-approval ja kein recheck beinhaltet)

DIR=$( dirname "$0" )

ANNOTATION_FILE="$DIR/../../config/testcases.annot"
TESTCASE_CACHE_FILE="$DIR/../../.cache/testcases"

# ---

function resetconsole {
        # http://superuser.com/questions/122911/bash-reset-and-clear-commands

        # clear

        # clears the console screen, but not the scrollback buffer
        # this is actually the escape code to "reset" the terminal
        echo -en "\ec"

        # clears the scrollback buffer, but not the console screen
        # screen content remains, and cursor position remains at its last position
        echo -en "\e[3J"
}

function showPatternHighlighted {
        A=()

        patterns=$( "$DIR"/allpatterns )
        for p in ${patterns[@]}; do
                A+=("$p");
        done

        out=""
        while IFS= read data; do
                out="$out$data"$'\n'
        done

        # Attention: "less" or "nano" does not work with Unicode for Asian languages :-(
        # Maybe this helps? http://serverfault.com/questions/414760/how-to-make-the-less-command-handle-utf-8 .
        # But then it still can't handle colors...
        echo "$out" | "$DIR"/highlighter "${A[@]}"
}

function question {
        if [ -f "$ANNOTATION_FILE" ]; then
                ANNOTS=$( cat "$ANNOTATION_FILE" | grep -E "^$query:" )
                if [ "$ANNOTS" != "" ]; then
                        echo ""
                        echo "********************"
                        echo "Notes in $ANNOTATION_FILE :"
                        OLDIFS="$IFS"
                        IFS=$'\n'
                        for p in ${ANNOTS[@]}; do
                                pos="${#query}"
                                (( pos += 2 ))
                                echo "   ${p:$pos}";
                        done
                        IFS="$OLDIFS"
                        echo "********************"
                fi
        fi
        echo ""
        echo "Keep in mind to only set this result as expected state, when:"
        echo "   (1) The output is well formatted OR when you add an entry to the ToDo"
        echo "       list when the output is 'OK, but not optimal'."
        echo "   (2) For web-requests you should also have a testcase which shows"
        echo "       non-existant domains"
        echo "   (3) All handles should have testcases too, or it should be noted that"
        echo "       specific handles cannot be queried by the NIC."
        echo "   (4) There should be no volative stuff like dates, serving whois server"
        echo "       names or your IP address in the output."
        echo "       Otherwise you should grep them away in $0 before you approve this testcase."
        echo "   (5) In notices, URLs/information must be correct"
        echo "   (6) Highlighting shall be working, otherwise the patterns or $DIR/highlighter needs to be changed"
        while true; do
                echo ""
                if [ -f "$E" ]; then
                        echo "Do you want to override the expected result? ([y]es, [n]o, [s]kip, [r]etry, [a]dd note, reset[t], e[x]it)"
                else
                        # TODO: braucht man hier reset?
                        echo "Do you want to define this as the expected result? ([y]es, [n]o, [s]kip, [r]etry, [a]dd note, rese[t], e[x]it)"
                fi
                read yn
                case $yn in
                        [Yy]* )
                                cat "$T" > "$E"
                                rm "$T"
                                if [ -f "$Q" ] && [ "$Q" != "$T" ]; then
                                        rm "$Q"
                                fi
                                touch "$tsfile"
                                return 0
                                ;;
                        [Nn]* )
                                rm "$T"
                                return 1
                                ;;
                        [Ss]* )
                                return 1
                                ;;
                        [Rr]* )
                                bakmode="$mode"
                                mode="i"
                                bakforce="$force"
                                force=1
                                process "$query"
                                RES=$?
                                mode="$bakmode"
                                force="$bakforce"
                                return $RES
                                ;;
                        [Aa]* )
                                echo "Please enter a note you want to add or empty string if you want to cancel"
                                read note

                                if [ "$note" == "" ]; then
                                        echo "Cancelled"
                                else
                                        echo "Note added to testcase of $query"
                                        echo "$query: $note" >> "$ANNOTATION_FILE"
                                fi
                                ;;
                        [Xx]* )
                                return 2
                                ;;
                        [Tt]* )
                                # Clears the expected state
                                #rm $TESTCASE_CACHE_FILE/*/$query
                                rm "$E"
                                rm "$Q"
                                rm "$tsfile"
                                # Now retry
                                # TODO: codeduplikat vermeiden?
                                bakmode="$mode"
                                mode="i"
                                bakforce="$force"
                                force=1
                                process "$query"
                                RES=$?
                                mode="$bakmode"
                                force="$bakforce"
                                return $RES
                                ;;
                        * )
                                echo "Please answer with the letter written in square brackets."
                                ;;
                esac
        done
}

function question2 {
        while true; do
                echo ""
                echo "vgwhois might already have been fixed. Do you want to enforce a vgwhois recheck now? ([y]es, [n]o, e[x]it)"
                read yn
                case $yn in
                        [YyRr]* )
                                if [ -f "$Q" ]; then
                                        rm "$Q"
                                fi
                                bakmode="$mode"
                                mode="i"
                                bakforce="$force"
                                force=1
                                process "$query"
                                RES=$?
                                mode="$bakmode"
                                force="$bakforce"
                                return $RES
                                ;;
                        [NnSs]* )
                                return 1
                                ;;
                        [Xx]* )
                                return 2
                                ;;
                        * )
                                echo "Please answer with the letter written in square brackets."
                                ;;
                esac
        done
}

function unquote {
        echo "$1" | sed "s/^'\(.*\)'$/\1/";
}

function process {
        if [ "$mode" != "b" ]; then
                # TODO: diese meldung soll nicht kommen, wenn einfach nur da steht "does not need a recheck" now...
                resetconsole
        fi

        query="$1"
        echo "Query: $query"

        E="$TESTCASE_CACHE_FILE/expected/$query"
        Q="$TESTCASE_CACHE_FILE/problems/$query"
        tsfile="$TESTCASE_CACHE_FILE/checktimestamps/$query"

        if [ "$mode" == "u" ]; then
                T="$Q"
                if [ ! -f "$T" ]; then
                        echo "There is no action needed by the user"
                        return 0
                fi
        else
                T=$( mktemp --suffix='.vgwhoisTC' )

                # In interactive/background mode, we will always do a webrequest when there is no problem and no expectation file without respect of the last checktime, to avoid that the status monitor will show entries with "no expectation file" when the user pressed "no", and then he would have to wait 7 days until "batch u" works again.
                # TODO: how to do that in 1 line?
                CALL_GWI=0
                if [ $force -eq 1 ]; then
                        CALL_GWI=1
                else
                        test \( ! -f "$Q" \) -a \( ! -f "$E" \)
                        if [ $? -eq 0 ]; then
                                CALL_GWI=1
                        elif [ ! -f "$tsfile" ] || [ $( stat --format=%Y "$tsfile" ) -le $(( $( date +%s ) - $recheck_time )) ]; then
                                if [ -f "$Q" ] && [ $( stat --format=%Y "$Q" ) -gt $(( $( date +%s ) - $recheck_time )) ]; then
                                        CALL_GWI=0
                                else
                                        CALL_GWI=1
                                fi
                        fi
                fi
                if [ $CALL_GWI -eq 1 ]; then
                        echo "... calling vgwhois ..."
                        # We have to use loc_vgwhois to allow $trytor to work correctly.
                        # The torifiers "torify" and "usewithtor" always outputting bogus "libtorsocks" warning messages which would be saved in the output
                        # "vtor" - if applied to this script - can only filter them from STDOUT and STDERR, but not intercept this "&>" pipe
                        # So we have to use this loc_vgwhois script, where we torify manually
                        # Also, it is important that we do the warning message filtering in this step, because in the final output the lines will be colored/highlighted, and therefore "vtor" cannot grep them correctly anymore.
                        # Note: regex only valid in the years 1000-1099, 1900-2099, 2900-2999
                        # grep away volative stuff like server names or times
                        (
                                "$DIR"/loc_vgwhois "$query" \
                                        | grep -v "This query was served by " \
                                        | grep -v "(c)[12][90][0-9]\{2\}" \
                                        | grep -v "[12][90][0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" \
                                        | grep -v "\[[0-9]\{1,3\}.[0-9]\{1,3\}.[0-9]\{1,3\}.[0-9]\{1,3\} REQUEST\]" \
                                        | grep -v "RL Net \[.*\] - RL IP \[.*\]" \
                                        | grep -v "% Query time:" \
                                        | grep -v "% request from" \
                                        | grep -v "% Last update of whois database:" \
                                        | grep -v "%AM TLD whois server #" \
                                        | grep -v "Last updated on " \
                                        | grep -v "This is the Ukrainian Whois query server #" \
                                        | grep -v "Last update of whois database" \
                                        | grep -v "Query time:" \
                                        | grep -v "nsstat:" \
                                        | grep -v "nslastaa:"
                        ) &> "$T"
                        # auskommentiert wegen problem: wenn man einen recheck wegen perl errors macht und dann abbricht, ist dann der PROBLEM state gespeichert? nein, er wurde im i-mode gelöscht!
                        # -> lösung ?rm "$Q" nur im erfolgsfall
                        #                       if [ -f "$Q" ]; then
                        #                               rm "$Q"
                        #                       fi

                        # behoben? in mode=i and mode=u, should the output of a new vgwhois request be saved into the problem file, if the user cancels? (to avoid a second web request?)
                        #       also, when doing a recheck after a perl error, and then cancel, the problem-file will not be updated!
                        # TODO: was ist besser?
                        #if [ ! -f "$Q" ] && [ "$T" != "$Q" ]; then
                        if [ "$T" != "$Q" ]; then
                                cat "$T" > "$Q"
                        fi


                else
                        rm "$T"
                        if [ "$mode" == "b" ]; then
                                echo "The query does not need a recheck now. Use --force to enforce it."
#                               if [ -f "$Q" ] && [ "$Q" != "$T" ]; then
#                                       rm "$Q"
#                               fi
                                return 0
                        else
                                T="$Q"
                                if [ ! -f "$T" ]; then
                                        echo "There is no action needed by the user"
                                        return 0
                                fi
                        fi
                fi
        fi

        WARNINGS=()
        cat "$T" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null
        if [ $? -eq 0 ]; then
                WARNINGS+=("Perl errors found in vgwhois output!")
        fi
        cat "$T" | head -n 1 | grep -E "^("$'\xEF\xBB\xBF'"){0,1}Process query: '$query'" > /dev/null
        if [ $? -ne 0 ]; then
                WARNINGS+=("The vgwhois output does not begin with 'Process query'!")
        fi
        cat "$T" | grep "vgwhois remarks: If this is a valid domainname or handle, please file a bug report." > /dev/null
        if [ $? -eq 0 ]; then
                WARNINGS+=("No pattern match!")
        fi
        if [ ${#WARNINGS[@]} -gt 0 ]; then
                if [ "$T" != "$Q" ]; then
                        cat "$T" > "$Q"
                        rm "$T"
                fi
#               if [ "$mode" != "b" ]; then
#                       resetconsole
#               fi
                OLDIFS="$IFS"
                IFS=$'\n'
                for p in ${WARNINGS[@]}; do
                        echo "WARNING: $p"
                done
                IFS="$OLDIFS"
                if [ "$mode" == "b" ]; then
                        echo "Saved for later analysis."
                        # TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!
#                       touch "$tsfile"
                        return 1
                else
                        # Hinweis: niemals vor "question" oder "question2" ein ts aktualisieren. denn wenn man in question* einen recheck beantragt und dann abbricht, dann würde dieser aktualisierte ts bewirken, dass der testcase erst wieder in 7 tagen angezeigt wird
                        echo ""
                        cat "$Q" | showPatternHighlighted
                        question2
                        return $?
                fi
        fi

        if [ -f "$E" ]; then
                D=$( date -r "$E" )
                echo "Compare with results of $D"
                diff -U 0 "$E" "$T" | showPatternHighlighted
                if [ ${PIPESTATUS[0]} -eq 0 ]; then
                        echo "OK! No differences found!"
                        rm "$T"
                        touch "$tsfile"
                        if [ -f "$Q" ] && [ "$Q" != "$T" ]; then
                                rm "$Q"
                        fi
                        return 0
                else
                        echo "Problem! Differences found!"

                        if [ "$mode" == "b" ]; then
                                echo "Saved for later analysis."
                                if [ "$T" != "$Q" ]; then
                                        cat "$T" > "$Q"
                                        rm "$T"
                                fi

                                # TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!
#                               touch "$tsfile"
                                return 1
                        else
                                if [ "$T" != "$Q" ]; then
                                        rm "$T"
                                fi

                                question
                                return $?
                        fi
                fi
        else
                if [ "$mode" == "b" ]; then
                        echo "This query has no expected state. Please define one."
                        echo "Saved for later analysis."
                        if [ "$T" != "$Q" ]; then
                                cat "$T" > "$Q"
                                rm "$T"
                        fi
                        # TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!
#                       touch "$tsfile"
                        return 1
                else
#                       resetconsole
                        echo "This query has no expected state. Please define one."
                        echo "This is the current output of vgwhois:"

                        echo ""
                        cat "$T" | showPatternHighlighted
                        echo ""

                        if [ "$T" != "$Q" ]; then
                                rm "$T"
                        fi

                        question
                        return $?
                fi
        fi
}

function HumanReadableTime {
        local seconds=$1
        local days=$(($seconds/86400))
        seconds=$(($seconds-($days*86400) ))
        local hours=$(($seconds/3600))
        seconds=$((seconds-($hours*3600) ))
        local minutes=$(($seconds/60))
        seconds=$(( $seconds-($minutes*60) ))

        # echo -n "${days}D ${hours}H ${minutes}M ${seconds}S"
        if [ $days -gt 0 ]; then
                if [ $hours -gt 12 ]; then
                        ((days++));
                fi
                echo -n "${days} days"
        elif [ $hours -gt 0 ]; then
                if [ $minutes -gt 30 ]; then
                        ((hours++));
                fi
                echo -n "${hours} hours"
        elif [ $minutes -gt 0 ]; then
                if [ $seconds -gt 30 ]; then
                        ((minutes++));
                fi
                echo -n "${minutes} minutes"
        else
                echo -n "a few seconds"
        fi
}

function usage {
        . "$DIR/../../config/testcases.conf"

        hf_recheck_time=$( HumanReadableTime $recheck_time )

        echo "Syntax: $0 options query"
        echo "  -h|--help"
        echo "  -m|--mode mode"
        echo "          Default: $mode"
        echo "          Mode: i = interactive (download and then show results/ask the developer if the result is OK)"
        echo "                b = background (download only and save the results for later query)"
        echo "                u = user-dialog only (only ask the developer if there are questions, e.g. which were generated by mode b)."
        echo "  -r|--rechecktime seconds"
        echo "          Default: $recheck_time = approx. $hf_recheck_time)"
        echo "  -f|--force"
        echo "          Ignores --rechecktime and forces a new vgwhois request"
        echo "          Default: $force"
}

# ---

if [ $# -eq 0 ]; then
        usage;
        exit;
fi

if [ ! -d "$TESTCASE_CACHE_FILE/checktimestamps" ]; then
        mkdir -p "$TESTCASE_CACHE_FILE/checktimestamps"
fi

if [ ! -d "$TESTCASE_CACHE_FILE/problems" ]; then
        mkdir -p "$TESTCASE_CACHE_FILE/problems"
fi

if [ ! -d "$TESTCASE_CACHE_FILE/expected" ]; then
        mkdir -p "$TESTCASE_CACHE_FILE/expected"
fi

# defaults
. "$DIR/../../config/testcases.conf"

PARAMS=( "$@" );
optarr=( $( getopt --name "$0" --options 'hfr:m:' --long 'help,force,rechecktime:,mode:' -- "${PARAMS[@]}" 2> /dev/null ) );

# Now process the arguments
i=0;
while true; do
        case "${optarr[$i]}" in
                -h|--help)
                        usage;
                        exit 0;
                        ;;
                -f|--force)
                        force=1;
                        ((i++));
                        ;;
                -r|--rechecktime)
                        recheck_time=$( unquote "${optarr[$((i+1))]}" );
                        ((i=i+2));
                        ;;
                -m|--mode)
                        mode=$( unquote "${optarr[$((i+1))]}" );

                        if [ "$mode" != "i" ] && [ "$mode" != "b" ] && [ "$mode" != "u" ]; then
                                echo "Invalid mode '$mode'"
                                usage
                                exit 2
                        fi

                        ((i=i+2));
                        ;;
                --)
                        ((i++));
                        break;
                        ;;
                *)
                        # Should never happen
                        echo "$0: Internal error while command-line-processing! Please report this error as bug." >&2;
                        exit 2;
                        ;;
        esac
done

RESTARGS=${optarr[@]:i}; # i..end
CMD="${optarr[i]}";
ARG="${optarr[@]:((i+1))}"; # (i+1)..end


EXITSTATUS=0
for X in ${RESTARGS[@]}
do
        X=$( unquote "$X" )
        process "$X"
        EC=$?

        if [ $EC -eq 2 ]; then
                # user pressed x for exit
                exit $EXITSTATUS;
        elif [ $EC -gt $EXITSTATUS ]; then
                # exitcode = max(exitcodes from processes)
                EXITSTATUS=$EC
        fi

done

exit $EXITSTATUS