Subversion Repositories vgwhois

Compare Revisions

No changes between revisions

Regard whitespace Rev 1 → Rev 2

/trunk/maintenance/qa-monitor/testcases/allpatterns
0,0 → 1,90
#!/usr/bin/php
<?php
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2014 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2014-11-25
#
 
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
 
$patterns = glob(__DIR__ . '/../../main/pattern/*');
 
#$bs = '\\b';
#$be = '\\b';
 
$vc = '[\p{L}a-zA-Z0-9_\\-\\+\\*\\.]';
$bs = '(?<!'.$vc.')';
$be = '(?!'.$vc.')';
 
foreach ($patterns as $pattern_file) {
$pattern = file($pattern_file);
 
$pattern[] = ':end';
 
$content = '';
foreach ($pattern as $p) {
$p = trim($p);
if ($p == '') continue;
if ($p[0] == '#') continue; // comment
 
if (preg_match('@^\\^redirect:@ismU', $p, $m)) continue;
 
$p = str_replace(array('(.*)', '(.+)'), '', $p);
 
if ($p[0] == ':') {
$content = '';
} else {
if ($p[0] == '=') {
// IP
$p_ = substr($p, 1);
 
// ignore for now
continue;
} else if ($p[0] == '*') {
// ASN
 
preg_match('#\*(.*):(\d+)(-(\d+)){0,1}#isU', $p, $m);
$prefix = $m[1];
$min = $m[2];
$max = (isset($m[4])) ? $m[4] : $min;
 
// ignore for now
continue;
} else if (preg_match('@^(urn:){0,1}oid:(.*)@i', $p, $m)) {
// OIDs
 
$regex = normalize_oid($m[2]);
$regex = str_replace('.', '\\.0*', $regex);
$regex = '@('.$bs.$regex.'(\\.\\d+)*'.$be.')@iU';
 
echo "$regex\n";
} else {
// REGEX
 
$regex = $p;
 
if ($regex == '.') continue; // last resort
 
$regex = preg_replace('@(?<!(?<!\\\\)\\$)$@', $vc.'*', $regex);
$regex = preg_replace('@(?<!\\\\)\\$$@', '', $regex);
 
$regex = preg_replace('@^(?!\\^)@', $vc.'*', $regex);
$regex = preg_replace('@^\\^@', '', $regex);
 
$regex = preg_replace('@(?>!\\$)$@', $vc.'*', $regex);
$regex = preg_replace('@\\$$@', '', $regex);
 
$regex = '@('.$bs.$regex.$be.')@iUu';
 
echo "$regex\n";
}
}
}
}
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/batch
0,0 → 1,55
#!/bin/bash
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2014 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2014-08-19
#
 
DIR=$( dirname "$0" )
 
ARGS=()
 
shuffle() {
# http://stackoverflow.com/a/5533586
local i tmp size max rand
 
# $RANDOM % (i+1) is biased because of the limited range of $RANDOM
# Compensate by using a range which is a multiple of the array size.
size = ${#ARGS[*]}
max = $(( 32768 / size * size ))
 
for ((i=size-1; i>0; i--)); do
while (( (rand=$RANDOM) >= max )); do :; done
rand=$(( rand % (i+1) ))
tmp=${ARGS[i]} ARGS[i]=${ARGS[rand]} ARGS[rand]=$tmp
done
}
 
while read f; do
if [ -z "$f" ]; then
continue;
fi
 
echo "$f" | grep -E "\s*^#" > /dev/null
if [ $? -eq 0 ]; then
continue;
fi
 
# Warum? "tld jp" wird dann zu "tld"
# query="$( basename $f )"
query="$f";
 
ARGS+=("$query")
done < "$DIR/../../config/testcases.list"
 
# TODO: does not work
#shuffle
 
"$DIR"/single $@ ${ARGS[@]}
exit $?
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/coverage
0,0 → 1,145
#!/usr/bin/php
<?php
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2014 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2014-11-25
#
 
# There are 2 possibilities
# - Coverage by section. At least 1 line per command must be covered
# => this is currently our behavior
# - Every line must be covered
 
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
 
$testcases = array();
$testcases_tmp = file(__DIR__ . '/../config/testcases.list');
foreach ($testcases_tmp as &$tc) {
$tc = trim($tc);
if ($tc == '') continue;
if ($tc[0] == '#') continue;
$testcases[] = $tc;
}
 
$patterns = glob(__DIR__ . '/../../main/pattern/*');
 
$count_total = 0;
$count_covered = 0;
$count_uncovered = 0;
 
echo "Uncovered\n\n";
 
foreach ($patterns as $pattern_file) {
$pattern = file($pattern_file);
 
$pattern[] = ':end';
$count_total--; // wegen ":end"
 
$content = '';
$covered = false;
$cmd = '';
foreach ($pattern as $p) {
$p = trim($p);
if ($p == '') continue;
if ($p[0] == '#') continue; // comment
 
if ($p[0] == ':') {
$count_total++;
if ($covered) {
$count_covered++;
$covered = false;
} else {
if ($cmd != '') {
$count_uncovered++;
echo "$pattern_file ($cmd): $content\n\n";
}
}
$cmd = $p;
$content = '';
} else if (!$covered) {
if ($p[0] == '=') {
// IP
 
$p_ = substr($p, 1);
 
foreach ($testcases as $query) {
$match = false;
if (strpos($p, ':') !== false) {
// IPv6
if (!ipv6_valid($query)) continue;
$covered = ipv6_in_cidr($p_, $query);
} else {
// IPv4
if (!ipv4_valid($query)) continue;
$covered = ipv4_in_cidr($p_, $query);
}
if ($covered) break;
}
} else if ($p[0] == '*') {
// ASN
 
preg_match('#\*(.*):(\d+)(-(\d+)){0,1}#isU', $p, $m);
$prefix = $m[1];
$min = $m[2];
$max = (isset($m[4])) ? $m[4] : $min;
 
foreach ($testcases as $query) {
if (preg_match('#'.preg_quote($prefix, '#').'(\d+)#is', $query, $m)) {
$num = $m[1];
if (($num >= $min) && ($num <= $max)) {
$covered = true;
break;
}
}
}
} else if (preg_match('@^(urn:){0,1}oid:(.*)@i', $p, $m)) {
// OIDs
 
$oid = normalize_oid($m[2]);
 
foreach ($testcases as $query) {
if (preg_match('@^(urn:){0,1}oid:(.*)@i', $query, $m2)) {
$oid_tc = normalize_oid($m2[2]);
 
if (strpos($oid.'.', $oid_tc.'.') === 0) {
$covered = true;
break;
}
}
}
} else {
// REGEX
 
$regex = $p;
 
foreach ($testcases as $query) {
if (preg_match('/'.$regex.'/i', $query, $m)) {
$covered = true;
break;
}
}
}
}
$content .= "$p\n";
}
}
 
if ($count_uncovered == 0) {
echo 'Every instruction is covered with at least one testcase!\n\n';
}
 
echo "Stats\n\n";
 
echo "Total = $count_total\n";
echo "Covered = $count_covered\n";
echo "Uncovered = $count_uncovered\n";
 
$coverage = $count_covered/$count_total * 100;
echo "Coverage = $coverage %\n";
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/highlighter
0,0 → 1,110
#!/usr/bin/php
<?php
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2015-04-17
#
 
#mb_internal_encoding('utf-8');
 
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/../../shared/php_includes/idna_convert.class.php';
 
$punycoder = new idna_convert();
 
$in = '';
while($f = fgets(STDIN)) {
$in .= $f;
}
 
$pre = generateRandomToken($in);
$post = generateRandomToken($in);
$plc1 = generateRandomToken($in);
 
// to avoid that "domain......ati.tn" will be highlighted completely.
$in = preg_replace('@\\.{2,}+@u', '<<<'.$plc1.':${0}>>>', $in);
 
if ($argc == 1) {
$out = array();
exec(__DIR__ . '/allpatterns', $out, $code);
if ($code != 0) {
echo __DIR__ . "/allpatterns failed with code $code\n"; # TODO: STDERR
exit(1);
}
foreach ($out as $o) {
$o = trim($o);
if ($o == '') continue;
$argv[] = $o;
}
}
 
array_shift($argv);
# $i = 0;
foreach ($argv as $x) {
# TODO: problem: $x darf nicht quoted werden, darf aber auch nicht @ enthalten -> dynamisch rausfinden welcher delimiter nicht vorkommt
# TODO: there are several problems using this mechanism. It is very hard to highlight everything which is queryable through gwhois.
# for example: (1) it is not possible to use \S+ in the pattern files, because it would match everything in the gwhois output, even though it was only meant to match everything in the query
# (2) if the rule .com.xx is applied after .xx , the .com.xx will not match, since .xx is already wrapped with $pre and $post
 
# $i++;
# $pre = "[start$i]";
# $post = "[end$i]";
 
$in = preg_replace($x, $pre.'${0}'.$post, $in);
 
$c = 0;
 
$x = preg_replace_callback("@(xn\-\-[0-9a-z\-]+)@ui", "punycodeCB", $x, -1, $c);
if ($c > 0) {
$in = preg_replace($x, $pre.'${0}'.$post, $in);
}
}
 
# Remove nesting (( x (( y )) a )) -> (( x a ))
do {
$ok = true;
$in = preg_replace_callback("@$pre(.*$post)@Uu", "verschachtelungCB", $in);
} while (!$ok);
 
# colorize
$in = str_replace($pre, "\033[41m\033[37m", $in);
$in = str_replace($post, "\033[0m", $in);
 
# undo it again (see above)
$in = preg_replace('@<<<'.$plc1.':([^>]+)>>>@u', '${1}', $in);
 
echo $in;
 
# ---
 
function verschachtelungCB($treffer) {
global $pre, $post, $ok;
 
if (strpos($treffer[1], $pre) === false) {
return $pre.$treffer[1];
} else {
$ok = false;
return $pre.str_replace(array($pre,$post), '', $treffer[1]);
}
}
 
function punycodeCB($treffer) {
global $punycoder;
 
$out = "";
$x = $punycoder->decode(strtolower($treffer[0]));
preg_match_all('/./u', $x, $results);
foreach ($results[0] as $m) {
$out .= '\\x{'.dechex(_uniord($m)).'}';
}
 
return $out;
}
 
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/loc_gwhois
0,0 → 1,34
#!/bin/bash
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2014 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2014-08-25
#
 
# TODO: $@ or $* ?
 
DIR=$( dirname "$0" )
 
. "$DIR/../../config/testcases.conf"
 
if [ $trytor -eq 1 ]; then
if [ -e "/usr/bin/vtor" ]; then
# We use vtor first, because it needs to cleanup temp files, which is only possible if it is not killed... Does this work?
vtor -a -r 1 -- "$DIR"/timeout.sh -t 10 -- gwhois $@
elif [ -e "/usr/bin/usewithtor" ]; then
"$DIR"/timeout.sh -t 10 -- usewithtor -- gwhois $@
elif [ -e "/usr/bin/torify" ]; then
"$DIR"/timeout.sh -t 10 -- torify -- gwhois $@
else
echo "Warning: Cannot find any TOR proxy software. Will do the query without TOR." >&2
"$DIR"/timeout.sh -t 10 -- gwhois $@
fi
else
"$DIR"/timeout.sh -t 10 -- gwhois $@
fi
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/single
0,0 → 1,542
#!/bin/bash
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2015-05-03
#
 
# TODO: use >&2
 
# TODO: anzeigen von wann der whois output ist... "last checked" -> "last activity" (da ein user-mode-batch-approval ja kein recheck beinhaltet)
 
DIR=$( dirname "$0" )
 
annotfile="$DIR/../../config/testcases.annot"
 
TESTCASE_CACHE_FILE="$DIR/../../.cache/testcases"
 
# ---
 
function resetconsole {
# http://superuser.com/questions/122911/bash-reset-and-clear-commands
 
# clear
 
# clears the console screen, but not the scrollback buffer
# this is actually the escape code to "reset" the terminal
echo -en "\ec"
 
# clears the scrollback buffer, but not the console screen
# screen content remains, and cursor position remains at its last position
echo -en "\e[3J"
}
 
function showPatternHighlighted {
A=()
 
patterns=$( "$DIR"/allpatterns )
for p in ${patterns[@]}; do
A+=("$p");
done
 
out=""
while IFS= read data; do
out="$out$data"$'\n'
done
 
# Attention: "less" or "nano" does not work with Unicode for Asian languages :-(
# Maybe this helps? http://serverfault.com/questions/414760/how-to-make-the-less-command-handle-utf-8 .
# But then it still can't handle colors...
echo "$out" | "$DIR"/highlighter "${A[@]}"
}
 
function question {
if [ -f "$annotfile" ]; then
ANNOTS=$( cat "$annotfile" | grep -E "^$query:" )
if [ "$ANNOTS" != "" ]; then
echo ""
echo "********************"
echo "Notes in $annotfile :"
OLDIFS="$IFS"
IFS=$'\n'
for p in ${ANNOTS[@]}; do
pos="${#query}"
(( pos += 2 ))
echo " ${p:$pos}";
done
IFS="$OLDIFS"
echo "********************"
fi
fi
echo ""
echo "Keep in mind to only set this result as expected state, when:"
echo " (1) The output is well formatted OR when you add an entry to the ToDo"
echo " list when the output is 'OK, but not optimal'."
echo " (2) For web-requests you should also have a testcase which shows"
echo " non-existant domains"
echo " (3) All handles should have testcases too, or it should be noted that"
echo " specific handles cannot be queried by the NIC."
echo " (4) There should be no volative stuff like dates, serving whois server"
echo " names or your IP address in the output."
echo " Otherwise you should grep them away in $0 before you approve this testcase."
echo " (5) In notices, URLs/information must be correct"
echo " (6) Highlighting shall be working, otherwise the patterns or $DIR/highlighter needs to be changed"
while true; do
echo ""
if [ -f "$E" ]; then
echo "Do you want to override the expected result? ([y]es, [n]o, [s]kip, [r]etry, [a]dd note, reset[t], e[x]it)"
else
# TODO: braucht man hier reset?
echo "Do you want to define this as the expected result? ([y]es, [n]o, [s]kip, [r]etry, [a]dd note, rese[t], e[x]it)"
fi
read yn
case $yn in
[Yy]* )
cat "$T" > "$E"
rm "$T"
if [ -f "$Q" ] && [ "$Q" != "$T" ]; then
rm "$Q"
fi
touch "$tsfile"
return 0
;;
[Nn]* )
rm "$T"
return 1
;;
[Ss]* )
return 1
;;
[Rr]* )
bakmode="$mode"
mode="i"
bakforce="$force"
force=1
process "$query"
RES=$?
mode="$bakmode"
force="$bakforce"
return $RES
;;
[Aa]* )
echo "Please enter a note you want to add or empty string if you want to cancel"
read note
 
if [ "$note" == "" ]; then
echo "Cancelled"
else
echo "Note added to testcase of $query"
echo "$query: $note" >> "$annotfile"
fi
;;
[Xx]* )
return 2
;;
[Tt]* )
# Clears the expected state
#rm $TESTCASE_CACHE_FILE/*/$query
rm "$E"
rm "$Q"
rm "$tsfile"
# Now retry
# TODO: codeduplikat vermeiden?
bakmode="$mode"
mode="i"
bakforce="$force"
force=1
process "$query"
RES=$?
mode="$bakmode"
force="$bakforce"
return $RES
;;
* )
echo "Please answer with the letter written in square brackets."
;;
esac
done
}
 
function question2 {
while true; do
echo ""
echo "Gwhois might already have been fixed. Do you want to enforce a gwhois recheck now? ([y]es, [n]o, e[x]it)"
read yn
case $yn in
[YyRr]* )
if [ -f "$Q" ]; then
rm "$Q"
fi
bakmode="$mode"
mode="i"
bakforce="$force"
force=1
process "$query"
RES=$?
mode="$bakmode"
force="$bakforce"
return $RES
;;
[NnSs]* )
return 1
;;
[Xx]* )
return 2
;;
* )
echo "Please answer with the letter written in square brackets."
;;
esac
done
}
 
function unquote {
echo "$1" | sed "s/^'\(.*\)'$/\1/";
}
 
function process {
if [ "$mode" != "b" ]; then
# TODO: diese meldung soll nicht kommen, wenn einfach nur da steht "does not need a recheck" now...
resetconsole
fi
 
query="$1"
echo "Query: $query"
 
E="$TESTCASE_CACHE_FILE/expected/$query"
Q="$TESTCASE_CACHE_FILE/problems/$query"
tsfile="$TESTCASE_CACHE_FILE/checktimestamps/$query"
 
if [ "$mode" == "u" ]; then
T="$Q"
if [ ! -f "$T" ]; then
echo "There is no action needed by the user"
return 0
fi
else
T=$( mktemp --suffix='.gwhoisTC' )
 
# In interactive/background mode, we will always do a webrequest when there is no problem and no expectation file without respect of the last checktime, to avoid that the status monitor will show entries with "no expectation file" when the user pressed "no", and then he would have to wait 7 days until "batch u" works again.
# TODO: how to do that in 1 line?
CALL_GWI=0
if [ $force -eq 1 ]; then
CALL_GWI=1
else
test \( ! -f "$Q" \) -a \( ! -f "$E" \)
if [ $? -eq 0 ]; then
CALL_GWI=1
elif [ ! -f "$tsfile" ] || [ $( stat --format=%Y "$tsfile" ) -le $(( $( date +%s ) - $recheck_time )) ]; then
if [ -f "$Q" ] && [ $( stat --format=%Y "$Q" ) -gt $(( $( date +%s ) - $recheck_time )) ]; then
CALL_GWI=0
else
CALL_GWI=1
fi
fi
fi
if [ $CALL_GWI -eq 1 ]; then
echo "... calling gwhois ..."
# We have to use loc_gwhois to allow $trytor to work correctly.
# The torifiers "torify" and "usewithtor" always outputting bogus "libtorsocks" warning messages which would be saved in the output
# "vtor" - if applied to this script - can only filter them from STDOUT and STDERR, but not intercept this "&>" pipe
# So we have to use this loc_gwhois script, where we torify manually
# Also, it is important that we do the warning message filtering in this step, because in the final output the lines will be colored/highlighted, and therefore "vtor" cannot grep them correctly anymore.
# Note: regex only valid in the years 1000-1099, 1900-2099, 2900-2999
# grep away volative stuff like server names or times
(
"$DIR"/loc_gwhois "$query" \
| grep -v "This query was served by " \
| grep -v "(c)[12][90][0-9]\{2\}" \
| grep -v "[12][90][0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" \
| grep -v "\[[0-9]\{1,3\}.[0-9]\{1,3\}.[0-9]\{1,3\}.[0-9]\{1,3\} REQUEST\]" \
| grep -v "RL Net \[.*\] - RL IP \[.*\]" \
| grep -v "% Query time:" \
| grep -v "% request from" \
| grep -v "% Last update of whois database:" \
| grep -v "%AM TLD whois server #" \
| grep -v "Last updated on " \
| grep -v "This is the Ukrainian Whois query server #" \
| grep -v "Last update of whois database" \
| grep -v "Query time:" \
| grep -v "nsstat:" \
| grep -v "nslastaa:"
) &> "$T"
# auskommentiert wegen problem: wenn man einen recheck wegen perl errors macht und dann abbricht, ist dann der PROBLEM state gespeichert? nein, er wurde im i-mode gelöscht!
# -> lösung ?rm "$Q" nur im erfolgsfall
# if [ -f "$Q" ]; then
# rm "$Q"
# fi
 
# behoben? in mode=i and mode=u, should the output of a new gwhois request be saved into the problem file, if the user cancels? (to avoid a second web request?)
# also, when doing a recheck after a perl error, and then cancel, the problem-file will not be updated!
# TODO: was ist besser?
#if [ ! -f "$Q" ] && [ "$T" != "$Q" ]; then
if [ "$T" != "$Q" ]; then
cat "$T" > "$Q"
fi
 
 
else
rm "$T"
if [ "$mode" == "b" ]; then
echo "The query does not need a recheck now. Use --force to enforce it."
# if [ -f "$Q" ] && [ "$Q" != "$T" ]; then
# rm "$Q"
# fi
return 0
else
T="$Q"
if [ ! -f "$T" ]; then
echo "There is no action needed by the user"
return 0
fi
fi
fi
fi
 
WARNINGS=()
cat "$T" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null
if [ $? -eq 0 ]; then
WARNINGS+=("Perl errors found in gwhois output!")
fi
cat "$T" | head -n 1 | grep -E "^("$'\xEF\xBB\xBF'"){0,1}Process query: '$query'" > /dev/null
if [ $? -ne 0 ]; then
WARNINGS+=("The gwhois output does not begin with 'Process query'!")
fi
cat "$T" | grep "gwhois remarks: If this is a valid domainname or handle, please file a bug report." > /dev/null
if [ $? -eq 0 ]; then
WARNINGS+=("No pattern match!")
fi
if [ ${#WARNINGS[@]} -gt 0 ]; then
if [ "$T" != "$Q" ]; then
cat "$T" > "$Q"
rm "$T"
fi
# if [ "$mode" != "b" ]; then
# resetconsole
# fi
OLDIFS="$IFS"
IFS=$'\n'
for p in ${WARNINGS[@]}; do
echo "WARNING: $p"
done
IFS="$OLDIFS"
if [ "$mode" == "b" ]; then
echo "Saved for later analysis."
# TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!
# touch "$tsfile"
return 1
else
# Hinweis: niemals vor "question" oder "question2" ein ts aktualisieren. denn wenn man in question* einen recheck beantragt und dann abbricht, dann würde dieser aktualisierte ts bewirken, dass der testcase erst wieder in 7 tagen angezeigt wird
echo ""
cat "$Q" | showPatternHighlighted
question2
return $?
fi
fi
 
if [ -f "$E" ]; then
D=$( date -r "$E" )
echo "Compare with results of $D"
diff -U 0 "$E" "$T" | showPatternHighlighted
if [ ${PIPESTATUS[0]} -eq 0 ]; then
echo "OK! No differences found!"
rm "$T"
touch "$tsfile"
if [ -f "$Q" ] && [ "$Q" != "$T" ]; then
rm "$Q"
fi
return 0
else
echo "Problem! Differences found!"
 
if [ "$mode" == "b" ]; then
echo "Saved for later analysis."
if [ "$T" != "$Q" ]; then
cat "$T" > "$Q"
rm "$T"
fi
 
# TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!
# touch "$tsfile"
return 1
else
if [ "$T" != "$Q" ]; then
rm "$T"
fi
 
question
return $?
fi
fi
else
if [ "$mode" == "b" ]; then
echo "This query has no expected state. Please define one."
echo "Saved for later analysis."
if [ "$T" != "$Q" ]; then
cat "$T" > "$Q"
rm "$T"
fi
# TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!!
# touch "$tsfile"
return 1
else
# resetconsole
echo "This query has no expected state. Please define one."
echo "This is the current output of gwhois:"
 
echo ""
cat "$T" | showPatternHighlighted
echo ""
 
if [ "$T" != "$Q" ]; then
rm "$T"
fi
 
question
return $?
fi
fi
}
 
function HumanReadableTime {
local seconds=$1
local days=$(($seconds/86400))
seconds=$(($seconds-($days*86400) ))
local hours=$(($seconds/3600))
seconds=$((seconds-($hours*3600) ))
local minutes=$(($seconds/60))
seconds=$(( $seconds-($minutes*60) ))
 
# echo -n "${days}D ${hours}H ${minutes}M ${seconds}S"
if [ $days -gt 0 ]; then
if [ $hours -gt 12 ]; then
((days++));
fi
echo -n "${days} days"
elif [ $hours -gt 0 ]; then
if [ $minutes -gt 30 ]; then
((hours++));
fi
echo -n "${hours} hours"
elif [ $minutes -gt 0 ]; then
if [ $seconds -gt 30 ]; then
((minutes++));
fi
echo -n "${minutes} minutes"
else
echo -n "a few seconds"
fi
}
 
function usage {
. "$DIR/../../config/testcases.conf"
 
hf_recheck_time=$( HumanReadableTime $recheck_time )
 
echo "Syntax: $0 options query"
echo " -h|--help"
echo " -m|--mode mode"
echo " Default: $mode"
echo " Mode: i = interactive (download and then show results/ask the developer if the result is OK)"
echo " b = background (download only and save the results for later query)"
echo " u = user-dialog only (only ask the developer if there are questions, e.g. which were generated by mode b)."
echo " -r|--rechecktime seconds"
echo " Default: $recheck_time = approx. $hf_recheck_time)"
echo " -f|--force"
echo " Ignores --rechecktime and forces a new gwhois request"
echo " Default: $force"
}
 
# ---
 
if [ $# -eq 0 ]; then
usage;
exit;
fi
 
if [ ! -d "$TESTCASE_CACHE_FILE/checktimestamps" ]; then
mkdir "$TESTCASE_CACHE_FILE/checktimestamps"
fi
 
if [ ! -d "$TESTCASE_CACHE_FILE/problems" ]; then
mkdir "$TESTCASE_CACHE_FILE/problems"
fi
 
if [ ! -d "$TESTCASE_CACHE_FILE/expected" ]; then
mkdir "$TESTCASE_CACHE_FILE/expected"
fi
 
# defaults
. "$DIR/../../config/testcases.conf"
 
PARAMS=( "$@" );
optarr=( $( getopt --name "$0" --options 'hfr:m:' --long 'help,force,rechecktime:,mode:' -- "${PARAMS[@]}" 2> /dev/null ) );
 
# Now process the arguments
i=0;
while true; do
case "${optarr[$i]}" in
-h|--help)
usage;
exit 0;
;;
-f|--force)
force=1;
((i++));
;;
-r|--rechecktime)
recheck_time=$( unquote "${optarr[$((i+1))]}" );
((i=i+2));
;;
-m|--mode)
mode=$( unquote "${optarr[$((i+1))]}" );
 
if [ "$mode" != "i" ] && [ "$mode" != "b" ] && [ "$mode" != "u" ]; then
echo "Invalid mode '$mode'"
usage
exit 2
fi
 
((i=i+2));
;;
--)
((i++));
break;
;;
*)
# Should never happen
echo "$0: Internal error while command-line-processing! Please report this error as bug." >&2;
exit 2;
;;
esac
done
 
RESTARGS=${optarr[@]:i}; # i..end
CMD="${optarr[i]}";
ARG="${optarr[@]:((i+1))}"; # (i+1)..end
 
 
EXITSTATUS=0
for X in ${RESTARGS[@]}
do
X=$( unquote "$X" )
process "$X"
EC=$?
 
if [ $EC -eq 2 ]; then
# user pressed x for exit
exit $EXITSTATUS;
elif [ $EC -gt $EXITSTATUS ]; then
# exitcode = max(exitcodes from processes)
EXITSTATUS=$EC
fi
 
done
 
exit $EXITSTATUS
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/status
0,0 → 1,121
#!/bin/bash
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2014 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2014-08-25
#
 
DIR=$( dirname "$0" )
 
# ---
 
function FileAge() {
echo $((`date +%s` - `stat -c %Y $1`))
}
 
function HumanReadableAge() {
local seconds=$1
local days=$(($seconds/86400))
seconds=$(($seconds-($days*86400) ))
local hours=$(($seconds/3600))
seconds=$((seconds-($hours*3600) ))
local minutes=$(($seconds/60))
seconds=$(( $seconds-($minutes*60) ))
 
# echo -n "${days}D ${hours}H ${minutes}M ${seconds}S"
if [ $days -gt 0 ]; then
if [ $hours -gt 12 ]; then
((days++));
fi
echo -n "${days} days ago"
elif [ $hours -gt 0 ]; then
if [ $minutes -gt 30 ]; then
((hours++));
fi
echo -n "${hours} hours ago"
elif [ $minutes -gt 0 ]; then
if [ $seconds -gt 30 ]; then
((minutes++));
fi
echo -n "${minutes} minutes ago"
else
echo -n "a few seconds ago"
fi
}
 
# ---
 
. "$DIR/../../config/testcases.conf"
 
echo "Query Last activity Status"
echo "----------------------------------------------------------------------------------"
 
while read f; do
if [ -z "$f" ]; then
continue;
fi
 
# Check if the line begins with an '#' (leading spaces are permitted)
echo "$f" | grep -E "\s*^#" > /dev/null
if [ $? -eq 0 ]; then
continue;
fi
 
# Warum? "tld jp" wird dann zu "tld"
# query="$( basename $f )"
query="$f";
 
expfile="$DIR/../../.cache/testcases/expected/$query"
 
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query"
if [ -f "$tsfile" ]; then
# lastcheck="$( date -r "$tsfile" )"
lastcheck=$( HumanReadableAge $( FileAge "$tsfile" ))
else
lastcheck="never"
fi
 
errfile="$DIR/../../.cache/testcases/problems/$query"
if [ -f "$errfile" ]; then
cat "$errfile" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null
PERLERR=$?
 
cat "$errfile" | head -n 1 | grep -E "^("$'\xEF\xBB\xBF'"){0,1}Process query: '$query'" > /dev/null
STARTEXP=$?
 
cat "$errfile" | grep "gwhois remarks: If this is a valid domainname or handle, please file a bug report." > /dev/null
NOPATTERNMATCH=$?
 
if [ $PERLERR -eq 0 ]; then
status="Perl-Error!"
elif [ $STARTEXP -ne 0 ]; then
status="Unexpected head line"
elif [ $NOPATTERNMATCH -eq 0 ]; then
status="No pattern match"
elif [ -f "$expfile" ]; then
status="Different output"
else
status="No expected output defined"
fi
else
if [ -f "$expfile" ]; then
status="OK"
else
status="No expected output defined"
fi
fi
 
if [ ! -f "$tsfile" ] || [ $( stat --format=%Y "$tsfile" ) -le $(( $( date +%s ) - $recheck_time )) ]; then
if [ ! -f "$errfile" ] || [ $( stat --format=%Y "$errfile" ) -le $(( $( date +%s ) - $recheck_time )) ]; then
status="$status, outdated"
fi
fi
 
printf "%-30s %-20s %-20s\n" "$query" "$lastcheck" "$status"
done < "$DIR/../../config/testcases.list"
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/status_short
0,0 → 1,53
#!/bin/bash
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2014 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2014-08-19
#
 
DIR=$( dirname "$0" )
 
. "$DIR/../../config/testcases.conf"
 
count=0
 
while read f; do
if [ -z "$f" ]; then
continue;
fi
 
echo "$f" | grep -E "\s*^#" > /dev/null
if [ $? -eq 0 ]; then
continue;
fi
 
# Warum? "tld jp" wird dann zu "tld"
# query="$( basename $f )"
query="$f";
 
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query"
errfile="$DIR/../../.cache/testcases/problems/$query"
 
if [ ! -f "$tsfile" ]; then
((count++));
elif [ ! -f "$tsfile" ] || [ $( stat --format=%Y "$tsfile" ) -le $(( $( date +%s ) - $recheck_time )) ]; then
((count++));
elif [ -f "$errfile" ]; then
((count++));
fi
 
done < "$DIR/../../config/testcases.list"
 
if [ $count -eq 0 ]; then
echo "[ OK ]"
exit 0
else
echo "[ !! ] $count testcases require attention"
exit 1
fi
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/testcases_cronrun
0,0 → 1,16
#!/bin/bash
 
#
# generic Whois - Maintenance Framework: Testcases
#
# (c) 2012-2014 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2014-04-08
#
 
DIR=$( dirname "$0" )
 
"$DIR"/batch --mode b &> /dev/null
Property changes:
Added: svn:executable
+*
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/timeout.sh
0,0 → 1,92
#!/bin/bash
#
# The Bash shell script executes a command with a time-out.
# Upon time-out expiration SIGTERM (15) is sent to the process. If the signal
# is blocked, then the subsequent SIGKILL (9) terminates it.
#
# Based on the Bash documentation example.
 
# Hello Chet,
# please find attached a "little easier" :-) to comprehend
# time-out example. If you find it suitable, feel free to include
# anywhere: the very same logic as in the original examples/scripts, a
# little more transparent implementation to my taste.
#
# Dmitry V Golovashkin <Dmitry.Golovashkin@sas.com>
 
scriptName="${0##*/}"
 
declare -i DEFAULT_TIMEOUT=9
declare -i DEFAULT_INTERVAL=1
declare -i DEFAULT_DELAY=1
 
# Timeout.
declare -i timeout=DEFAULT_TIMEOUT
# Interval between checks if the process is still alive.
declare -i interval=DEFAULT_INTERVAL
# Delay between posting the SIGTERM signal and destroying the process by SIGKILL.
declare -i delay=DEFAULT_DELAY
 
function printUsage() {
cat <<EOF
 
Synopsis
$scriptName [-t timeout] [-i interval] [-d delay] command
Execute a command with a time-out.
Upon time-out expiration SIGTERM (15) is sent to the process. If SIGTERM
signal is blocked, then the subsequent SIGKILL (9) terminates it.
 
-t timeout
Number of seconds to wait for command completion.
Default value: $DEFAULT_TIMEOUT seconds.
 
-i interval
Interval between checks if the process is still alive.
Positive integer, default value: $DEFAULT_INTERVAL seconds.
 
-d delay
Delay between posting the SIGTERM signal and destroying the
process by SIGKILL. Default value: $DEFAULT_DELAY seconds.
 
As of today, Bash does not support floating point arithmetic (sleep does),
therefore all delay/time values must be integers.
EOF
}
 
# Options.
while getopts ":t:i:d:" option; do
case "$option" in
t) timeout=$OPTARG ;;
i) interval=$OPTARG ;;
d) delay=$OPTARG ;;
*) printUsage; exit 1 ;;
esac
done
shift $((OPTIND - 1))
 
# $# should be at least 1 (the command to execute), however it may be strictly
# greater than 1 if the command itself has options.
if (($# == 0 || interval <= 0)); then
printUsage
exit 1
fi
 
# kill -0 pid Exit code indicates if a signal may be sent to $pid process.
(
((t = timeout))
 
while ((t > 0)); do
sleep $interval
kill -0 $$ || exit 0
((t -= interval))
done
 
# Be nice, post SIGTERM first.
# The 'exit 0' below will be executed if any preceeding command fails.
kill -s SIGTERM $$ && kill -0 $$ || exit 0
sleep $delay
kill -s SIGKILL $$
) 2> /dev/null &
 
exec "$@"
 
Property changes:
Added: svn:executable
+*
\ No newline at end of property
Added: svn:mime-type
+text/x-sh
\ No newline at end of property
/trunk/maintenance/qa-monitor/testcases/todo
0,0 → 1,78
Process query: '2001:0000:4136:e378:8000:63bf:3fff:fdd2'
 
Query recognized as IPv6.
Address expanded to '2001:0000:4136:e378:8000:63bf:3fff:fdd2'.
Querying script .../main/subprograms/teredo_decoder
 
00:06:00 libtorsocks(5179): sendto: Connection is a UDP or ICMP stream, may be a DNS request or other form of leak: rejecting.
00:06:00 libtorsocks(5179): sendto: Connection is a UDP or ICMP stream, may be a DNS request or other form of leak: rejecting.
00:06:00 libtorsocks(5179): sendto: Connection is a UDP or ICMP stream, may be a DNS request or other form of leak: rejecting.
00:06:00 libtorsocks(5179): sendto: Connection is a UDP or ICMP stream, may be a DNS request or other form of leak: rejecting.
00:06:00 libtorsocks(5179): sendto: Connection is a UDP or ICMP stream, may be a DNS request or other form of leak: rejecting.
00:06:00 libtorsocks(5179): sendto: Connection is a UDP or ICMP stream, may be a DNS request or other form of leak: rejecting.
Teredo IPv6 address: 2001:0000:4136:e378:8000:63bf:3fff:fdd2
Teredo prefix: 2001:0000
 
---
 
handle ...-SITA? (bei .aero)
 
---
 
highlighter: viele highlights funktionieren nicht und markieren bis zum ende oder bis zum anfang der zeile
 
---
 
highlighter brauchen testcases
 
möglichkeit alles zu vergessen, also
rm .cache/testcases/*/$query
und danach retry
 
ability to remove notes in the GUI (select note by entering a number)
 
GROSSES PROBLEM
obwohl ich den cron jede nacht laufen lasse, muss ich jedes mal bei "batch --mode b" warten
die "problem" timestamps sind 3 tage alt... warum?
 
---
 
TESTCASE GENERATOR TOOL:
for all missing testcase TLDs, try following to find a valid test domain:
- when whois server or url ends with <tld> , use this domain (without subdomains)
- try to do a nslookup to nic.<tld>
- search for site:*.<tid> at google
=> then add a testcase!
 
measure coverage mit ALLEN pattern files; line-coverage
 
weitere funktionen:
- remove notes from that testcase
- do a side query
 
arin sendet escape zeichen? z.B. bei "AS1", deswegen wir kein console scrollback buffer geleert!
-> nein, der console buffer ist einfach zu klein! aber less geht nicht mit color!
 
./single --mode u ati.tn
Query: ati.tn
There is no action needed by the user
-> es gibt keine problem file und keine expectation file. er sollte runterladen?
-> nein, aber meldung ist verwirrend. "mode u" sollte niemals inet conn haben
 
möglichkeiteben, die testcases mit attributen zu versehen
jedem testcase einen namen geben
jedem testcase eine möglichkeit geben, dinge per SED-REGEX auszublenden (z.b. ipadresse, uhrzeit etc).
 
einen befehl machen, der den expected status nochmal im korrekten highlight anzeigt. zum kontrollieren. damit man alle expectations nochmal sehen kann. (batch)
 
wenn man währendes diff (?) einen strg+c macht, dann kommt "no changes" - speichern?
 
----
 
gwi im allgemeinen : kann man einen unicode domainnamen eingeben? ggf auch als utf-8 ?
 
---
 
beschreibung, wie man dieses tool benutzt, also dass man ./batch aufrufen muss
 
notizen entfernbar machen (mit nano editieren)