Subversion Repositories vgwhois

#!/bin/bash

#

#  VGWhoIs (ViaThinkSoft Global WhoIs, a fork of generic Whois / gwhois)

#  Maintenance / Developer utilities

#

#  (c) 2012-2019 by Daniel Marschall, ViaThinkSoft <info@daniel-marschall.de>

#

#  License: https://www.gnu.org/licenses/gpl-2.0.html (GPL version 2)

#

# TODO: use >&2

# TODO: anzeigen von wann der whois output ist... "last checked" -> "last activity" (da ein user-mode-batch-approval ja kein recheck beinhaltet)

DIR=$( dirname "$0" )

ANNOTATION_FILE="$DIR/../../config/testcases.annot"

TESTCASE_CACHE_FILE="$DIR/../../.cache/testcases"

# ---

function resetconsole {

	# http://superuser.com/questions/122911/bash-reset-and-clear-commands

	# clear

	# clears the console screen, but not the scrollback buffer

	# this is actually the escape code to "reset" the terminal

	echo -en "\ec"

	# clears the scrollback buffer, but not the console screen

	# screen content remains, and cursor position remains at its last position

	echo -en "\e[3J"

}

function showPatternHighlighted {

	A=()

	patterns=$( "$DIR"/allpatterns )

	for p in ${patterns[@]}; do

		A+=("$p");

	done

	out=""

	while IFS= read data; do

		out="$out$data"$'\n'

	done

	# Attention: "less" or "nano" does not work with Unicode for Asian languages :-(

	# Maybe this helps? http://serverfault.com/questions/414760/how-to-make-the-less-command-handle-utf-8 .

	# But then it still can't handle colors...

	echo "$out" | "$DIR"/highlighter "${A[@]}"

}

function question {

	if [ -f "$ANNOTATION_FILE" ]; then

		ANNOTS=$( cat "$ANNOTATION_FILE" | grep -E "^$query:" )

		if [ "$ANNOTS" != "" ]; then

			echo ""

			echo "********************"

			echo "Notes in $ANNOTATION_FILE :"

			OLDIFS="$IFS"

			IFS=$'\n'

			for p in ${ANNOTS[@]}; do

				pos="${#query}"

				(( pos += 2 ))

				echo "   ${p:$pos}";

			done

			IFS="$OLDIFS"

			echo "********************"

		fi

	fi

	echo ""

	echo "Keep in mind to only set this result as expected state, when:"

	echo "   (1) The output is well formatted OR when you add an entry to the ToDo"

	echo "       list when the output is 'OK, but not optimal'."

	echo "   (2) For web-requests you should also have a testcase which shows"

	echo "       non-existant domains"

	echo "   (3) All handles should have testcases too, or it should be noted that"

	echo "       specific handles cannot be queried by the NIC."

	echo "   (4) There should be no volative stuff like dates, serving whois server"

	echo "       names or your IP address in the output."

	echo "       Otherwise you should grep them away in $0 before you approve this testcase."

	echo "   (5) In notices, URLs/information must be correct"

	echo "   (6) Highlighting shall be working, otherwise the patterns or $DIR/highlighter needs to be changed"

	while true; do

		echo ""

		if [ -f "$E" ]; then

			echo "Do you want to override the expected result? ([y]es, [n]o, [s]kip, [r]etry, [a]dd note, reset[t], e[x]it)"

		else

			# TODO: braucht man hier reset?

			echo "Do you want to define this as the expected result? ([y]es, [n]o, [s]kip, [r]etry, [a]dd note, rese[t], e[x]it)"

		fi

		read yn

		case $yn in

			[Yy]* )

				cat "$T" > "$E"

				rm "$T"

				if [ -f "$Q" ] && [ "$Q" != "$T" ]; then

					rm "$Q"

				fi

				touch "$tsfile"

				return 0

				;;

			[Nn]* )

				rm "$T"

				return 1

				;;

			[Ss]* )

				return 1

				;;

			[Rr]* )

				bakmode="$mode"

				mode="i"

				bakforce="$force"

				force=1

				process "$query"

				RES=$?

				mode="$bakmode"

				force="$bakforce"

				return $RES

				;;

			[Aa]* )

				echo "Please enter a note you want to add or empty string if you want to cancel"

				read note

				if [ "$note" == "" ]; then

					echo "Cancelled"

				else

					echo "Note added to testcase of $query"

					echo "$query: $note" >> "$ANNOTATION_FILE"

				fi

				;;

			[Xx]* )

				return 2

				;;

			[Tt]* )

				# Clears the expected state

				#rm $TESTCASE_CACHE_FILE/*/$query

				rm "$E"

				rm "$Q"

				rm "$tsfile"

				# Now retry

				# TODO: codeduplikat vermeiden?

				bakmode="$mode"

				mode="i"

				bakforce="$force"

				force=1

				process "$query"

				RES=$?

				mode="$bakmode"

				force="$bakforce"

				return $RES

				;;

			* )

				echo "Please answer with the letter written in square brackets."

				;;

		esac

	done

}

function question2 {

	while true; do

		echo ""

		echo "vgwhois might already have been fixed. Do you want to enforce a vgwhois recheck now? ([y]es, [n]o, e[x]it)"

		read yn

		case $yn in

			[YyRr]* )

				if [ -f "$Q" ]; then

					rm "$Q"

				fi

				bakmode="$mode"

				mode="i"

				bakforce="$force"

				force=1

				process "$query"

				RES=$?

				mode="$bakmode"

				force="$bakforce"

				return $RES

				;;

			[NnSs]* )

				return 1

				;;

			[Xx]* )

				return 2

				;;

			* )

				echo "Please answer with the letter written in square brackets."

				;;

		esac

	done

}

function unquote {

	echo "$1" | sed "s/^'\(.*\)'$/\1/";

}

function process {

	if [ "$mode" != "b" ]; then

		# TODO: diese meldung soll nicht kommen, wenn einfach nur da steht "does not need a recheck" now...

		resetconsole

	fi

	query="$1"

	echo "Query: $query"

	E="$TESTCASE_CACHE_FILE/expected/$query"

	Q="$TESTCASE_CACHE_FILE/problems/$query"

	tsfile="$TESTCASE_CACHE_FILE/checktimestamps/$query"

	if [ "$mode" == "u" ]; then

		T="$Q"

		if [ ! -f "$T" ]; then

			echo "There is no action needed by the user"

			return 0

		fi

	else

		T=$( mktemp --suffix='.vgwhoisTC' )

		# In interactive/background mode, we will always do a webrequest when there is no problem and no expectation file without respect of the last checktime, to avoid that the status monitor will show entries with "no expectation file" when the user pressed "no", and then he would have to wait 7 days until "batch u" works again.

		# TODO: how to do that in 1 line?

		CALL_GWI=0

		if [ $force -eq 1 ]; then

			CALL_GWI=1

		else

			test \( ! -f "$Q" \) -a \( ! -f "$E" \)

			if [ $? -eq 0 ]; then

				CALL_GWI=1

			elif [ ! -f "$tsfile" ] || [ $( stat --format=%Y "$tsfile" ) -le $(( $( date +%s ) - $recheck_time )) ]; then

				if [ -f "$Q" ] && [ $( stat --format=%Y "$Q" ) -gt $(( $( date +%s ) - $recheck_time )) ]; then

					CALL_GWI=0

				else

					CALL_GWI=1

				fi

			fi

		fi

		if [ $CALL_GWI -eq 1 ]; then

			echo "... calling vgwhois ..."

			# We have to use loc_vgwhois to allow $trytor to work correctly.

			# The torifiers "torify" and "usewithtor" always outputting bogus "libtorsocks" warning messages which would be saved in the output

			# "vtor" - if applied to this script - can only filter them from STDOUT and STDERR, but not intercept this "&>" pipe

			# So we have to use this loc_vgwhois script, where we torify manually

			# Also, it is important that we do the warning message filtering in this step, because in the final output the lines will be colored/highlighted, and therefore "vtor" cannot grep them correctly anymore.

			# Note: regex only valid in the years 1000-1099, 1900-2099, 2900-2999

			# grep away volative stuff like server names or times

			(

				"$DIR"/loc_vgwhois "$query" \

					| grep -v "This query was served by " \

					| grep -v "(c)[12][90][0-9]\{2\}" \

					| grep -v "[12][90][0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" \

					| grep -v "\[[0-9]\{1,3\}.[0-9]\{1,3\}.[0-9]\{1,3\}.[0-9]\{1,3\} REQUEST\]" \

					| grep -v "RL Net \[.*\] - RL IP \[.*\]" \

					| grep -v "% Query time:" \

					| grep -v "% request from" \

					| grep -v "% Last update of whois database:" \

					| grep -v "%AM TLD whois server #" \

					| grep -v "Last updated on " \

					| grep -v "This is the Ukrainian Whois query server #" \

					| grep -v "Last update of whois database" \

					| grep -v "Query time:" \

					| grep -v "nsstat:" \

					| grep -v "nslastaa:"

			) &> "$T"

			# auskommentiert wegen problem: wenn man einen recheck wegen perl errors macht und dann abbricht, ist dann der PROBLEM state gespeichert? nein, er wurde im i-mode gelöscht!

			# -> lösung ?rm "$Q" nur im erfolgsfall

			#			if [ -f "$Q" ]; then

			#				rm "$Q"

			#			fi

			# behoben? in mode=i and mode=u, should the output of a new vgwhois request be saved into the problem file, if the user cancels? (to avoid a second web request?)

			#       also, when doing a recheck after a perl error, and then cancel, the problem-file will not be updated!

			# TODO: was ist besser?

			#if [ ! -f "$Q" ] && [ "$T" != "$Q" ]; then

			if [ "$T" != "$Q" ]; then

				cat "$T" > "$Q"

			fi

		else

			rm "$T"

			if [ "$mode" == "b" ]; then

				echo "The query does not need a recheck now. Use --force to enforce it."

#				if [ -f "$Q" ] && [ "$Q" != "$T" ]; then

#					rm "$Q"

#				fi

				return 0

			else

				T="$Q"

				if [ ! -f "$T" ]; then

					echo "There is no action needed by the user"

					return 0

				fi

			fi

		fi

	fi

	WARNINGS=()

	cat "$T" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null

	if [ $? -eq 0 ]; then

		WARNINGS+=("Perl errors found in vgwhois output!")

	fi

	cat "$T" | head -n 1 | grep -E "^("$'\xEF\xBB\xBF'"){0,1}Process query: '$query'" > /dev/null

	if [ $? -ne 0 ]; then

		WARNINGS+=("The vgwhois output does not begin with 'Process query'!")

	fi

	cat "$T" | grep "vgwhois remarks: If this is a valid domainname or handle, please file a bug report." > /dev/null

	if [ $? -eq 0 ]; then

		WARNINGS+=("No pattern match!")

	fi

	if [ ${#WARNINGS[@]} -gt 0 ]; then

		if [ "$T" != "$Q" ]; then

			cat "$T" > "$Q"

			rm "$T"

		fi

#		if [ "$mode" != "b" ]; then

#			resetconsole

#		fi

		OLDIFS="$IFS"

		IFS=$'\n'

		for p in ${WARNINGS[@]}; do

			echo "WARNING: $p"

		done

		IFS="$OLDIFS"

		if [ "$mode" == "b" ]; then

			echo "Saved for later analysis."

			# TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!

#			touch "$tsfile"

			return 1

		else

			# Hinweis: niemals vor "question" oder "question2" ein ts aktualisieren. denn wenn man in question* einen recheck beantragt und dann abbricht, dann würde dieser aktualisierte ts bewirken, dass der testcase erst wieder in 7 tagen angezeigt wird

			echo ""

			cat "$Q" | showPatternHighlighted

			question2

			return $?

		fi

	fi

	if [ -f "$E" ]; then

		D=$( date -r "$E" )

		echo "Compare with results of $D"

		diff -U 0 "$E" "$T" | showPatternHighlighted

		if [ ${PIPESTATUS[0]} -eq 0 ]; then

			echo "OK! No differences found!"

			rm "$T"

			touch "$tsfile"

			if [ -f "$Q" ] && [ "$Q" != "$T" ]; then

				rm "$Q"

			fi

			return 0

		else

			echo "Problem! Differences found!"

			if [ "$mode" == "b" ]; then

				echo "Saved for later analysis."

				if [ "$T" != "$Q" ]; then

					cat "$T" > "$Q"

					rm "$T"

				fi

				# TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!

#				touch "$tsfile"

				return 1

			else

				if [ "$T" != "$Q" ]; then

					rm "$T"

				fi

				question

				return $?

			fi

		fi

	else

		if [ "$mode" == "b" ]; then

			echo "This query has no expected state. Please define one."

			echo "Saved for later analysis."

			if [ "$T" != "$Q" ]; then

				cat "$T" > "$Q"

				rm "$T"

			fi

			# TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!

#			touch "$tsfile"

			return 1

		else

#			resetconsole

			echo "This query has no expected state. Please define one."

			echo "This is the current output of vgwhois:"

			echo ""

			cat "$T" | showPatternHighlighted

			echo ""

			if [ "$T" != "$Q" ]; then

				rm "$T"

			fi

			question

			return $?

		fi

	fi

}

function HumanReadableTime {

	local seconds=$1

	local days=$(($seconds/86400))

	seconds=$(($seconds-($days*86400) ))

	local hours=$(($seconds/3600))

	seconds=$((seconds-($hours*3600) ))

	local minutes=$(($seconds/60))

	seconds=$(( $seconds-($minutes*60) ))

	# echo -n "${days}D ${hours}H ${minutes}M ${seconds}S"

	if [ $days -gt 0 ]; then

		if [ $hours -gt 12 ]; then

			((days++));

		fi

		echo -n "${days} days"

	elif [ $hours -gt 0 ]; then

		if [ $minutes -gt 30 ]; then

			((hours++));

		fi

		echo -n "${hours} hours"

	elif [ $minutes -gt 0 ]; then

		if [ $seconds -gt 30 ]; then

			((minutes++));

		fi

		echo -n "${minutes} minutes"

	else

		echo -n "a few seconds"

	fi

}

function usage {

	. "$DIR/../../config/testcases.conf"

	hf_recheck_time=$( HumanReadableTime $recheck_time )

	echo "Syntax: $0 options query"

	echo "	-h|--help"

	echo "	-m|--mode mode"

	echo "		Default: $mode"

	echo "		Mode: i = interactive (download and then show results/ask the developer if the result is OK)"

	echo "		      b = background (download only and save the results for later query)"

	echo "		      u = user-dialog only (only ask the developer if there are questions, e.g. which were generated by mode b)."

	echo "	-r|--rechecktime seconds"

	echo "		Default: $recheck_time = approx. $hf_recheck_time)"

	echo "	-f|--force"

	echo "		Ignores --rechecktime and forces a new vgwhois request"

	echo "		Default: $force"

}

# ---

if [ $# -eq 0 ]; then

	usage;

	exit;

fi

if [ ! -d "$TESTCASE_CACHE_FILE/checktimestamps" ]; then

	mkdir -p "$TESTCASE_CACHE_FILE/checktimestamps"

fi

if [ ! -d "$TESTCASE_CACHE_FILE/problems" ]; then

	mkdir -p "$TESTCASE_CACHE_FILE/problems"

fi

if [ ! -d "$TESTCASE_CACHE_FILE/expected" ]; then

	mkdir -p "$TESTCASE_CACHE_FILE/expected"

fi

# defaults

. "$DIR/../../config/testcases.conf"

PARAMS=( "$@" );

optarr=( $( getopt --name "$0" --options 'hfr:m:' --long 'help,force,rechecktime:,mode:' -- "${PARAMS[@]}" 2> /dev/null ) );

# Now process the arguments

i=0;

while true; do

	case "${optarr[$i]}" in

		-h|--help)

			usage;

			exit 0;

			;;

		-f|--force)

			force=1;

			((i++));

			;;

		-r|--rechecktime)

			recheck_time=$( unquote "${optarr[$((i+1))]}" );

			((i=i+2));

			;;

		-m|--mode)

			mode=$( unquote "${optarr[$((i+1))]}" );

			if [ "$mode" != "i" ] && [ "$mode" != "b" ] && [ "$mode" != "u" ]; then

				echo "Invalid mode '$mode'"

				usage

				exit 2

			fi

			((i=i+2));

			;;

		--)

			((i++));

			break;

			;;

		*)

			# Should never happen

			echo "$0: Internal error while command-line-processing! Please report this error as bug." >&2;

			exit 2;

			;;

	esac

done

RESTARGS=${optarr[@]:i}; # i..end

CMD="${optarr[i]}";

ARG="${optarr[@]:((i+1))}"; # (i+1)..end

EXITSTATUS=0

for X in ${RESTARGS[@]}

do

	X=$( unquote "$X" )

	process "$X"

	EC=$?

	if [ $EC -eq 2 ]; then

		# user pressed x for exit

		exit $EXITSTATUS;

	elif [ $EC -gt $EXITSTATUS ]; then

		# exitcode = max(exitcodes from processes)

		EXITSTATUS=$EC

	fi

done

exit $EXITSTATUS