Rev 3 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
2 | daniel-mar | 1 | #!/bin/bash |
2 | |||
3 | # |
||
4 | # generic Whois - Maintenance Framework: Testcases |
||
5 | # |
||
6 | # (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
||
7 | # |
||
8 | # Distribution, usage etc. pp. regulated by the current version of GPL. |
||
9 | # |
||
10 | # |
||
11 | # Version 2015-05-03 |
||
12 | # |
||
13 | |||
14 | # TODO: use >&2 |
||
15 | |||
16 | # TODO: anzeigen von wann der whois output ist... "last checked" -> "last activity" (da ein user-mode-batch-approval ja kein recheck beinhaltet) |
||
17 | |||
18 | DIR=$( dirname "$0" ) |
||
19 | |||
4 | daniel-mar | 20 | ANNOTATION_FILE="$DIR/../../config/testcases.annot" |
2 | daniel-mar | 21 | TESTCASE_CACHE_FILE="$DIR/../../.cache/testcases" |
22 | |||
23 | # --- |
||
24 | |||
25 | function resetconsole { |
||
26 | # http://superuser.com/questions/122911/bash-reset-and-clear-commands |
||
27 | |||
28 | # clear |
||
29 | |||
30 | # clears the console screen, but not the scrollback buffer |
||
31 | # this is actually the escape code to "reset" the terminal |
||
32 | echo -en "\ec" |
||
33 | |||
34 | # clears the scrollback buffer, but not the console screen |
||
35 | # screen content remains, and cursor position remains at its last position |
||
36 | echo -en "\e[3J" |
||
37 | } |
||
38 | |||
39 | function showPatternHighlighted { |
||
40 | A=() |
||
41 | |||
42 | patterns=$( "$DIR"/allpatterns ) |
||
43 | for p in ${patterns[@]}; do |
||
44 | A+=("$p"); |
||
45 | done |
||
46 | |||
47 | out="" |
||
48 | while IFS= read data; do |
||
49 | out="$out$data"$'\n' |
||
50 | done |
||
51 | |||
52 | # Attention: "less" or "nano" does not work with Unicode for Asian languages :-( |
||
53 | # Maybe this helps? http://serverfault.com/questions/414760/how-to-make-the-less-command-handle-utf-8 . |
||
54 | # But then it still can't handle colors... |
||
55 | echo "$out" | "$DIR"/highlighter "${A[@]}" |
||
56 | } |
||
57 | |||
58 | function question { |
||
4 | daniel-mar | 59 | if [ -f "$ANNOTATION_FILE" ]; then |
60 | ANNOTS=$( cat "$ANNOTATION_FILE" | grep -E "^$query:" ) |
||
2 | daniel-mar | 61 | if [ "$ANNOTS" != "" ]; then |
62 | echo "" |
||
63 | echo "********************" |
||
4 | daniel-mar | 64 | echo "Notes in $ANNOTATION_FILE :" |
2 | daniel-mar | 65 | OLDIFS="$IFS" |
66 | IFS=$'\n' |
||
67 | for p in ${ANNOTS[@]}; do |
||
68 | pos="${#query}" |
||
69 | (( pos += 2 )) |
||
70 | echo " ${p:$pos}"; |
||
71 | done |
||
72 | IFS="$OLDIFS" |
||
73 | echo "********************" |
||
74 | fi |
||
75 | fi |
||
76 | echo "" |
||
77 | echo "Keep in mind to only set this result as expected state, when:" |
||
78 | echo " (1) The output is well formatted OR when you add an entry to the ToDo" |
||
79 | echo " list when the output is 'OK, but not optimal'." |
||
80 | echo " (2) For web-requests you should also have a testcase which shows" |
||
81 | echo " non-existant domains" |
||
82 | echo " (3) All handles should have testcases too, or it should be noted that" |
||
83 | echo " specific handles cannot be queried by the NIC." |
||
84 | echo " (4) There should be no volative stuff like dates, serving whois server" |
||
85 | echo " names or your IP address in the output." |
||
86 | echo " Otherwise you should grep them away in $0 before you approve this testcase." |
||
87 | echo " (5) In notices, URLs/information must be correct" |
||
88 | echo " (6) Highlighting shall be working, otherwise the patterns or $DIR/highlighter needs to be changed" |
||
89 | while true; do |
||
90 | echo "" |
||
91 | if [ -f "$E" ]; then |
||
92 | echo "Do you want to override the expected result? ([y]es, [n]o, [s]kip, [r]etry, [a]dd note, reset[t], e[x]it)" |
||
93 | else |
||
94 | # TODO: braucht man hier reset? |
||
95 | echo "Do you want to define this as the expected result? ([y]es, [n]o, [s]kip, [r]etry, [a]dd note, rese[t], e[x]it)" |
||
96 | fi |
||
97 | read yn |
||
98 | case $yn in |
||
99 | [Yy]* ) |
||
100 | cat "$T" > "$E" |
||
101 | rm "$T" |
||
102 | if [ -f "$Q" ] && [ "$Q" != "$T" ]; then |
||
103 | rm "$Q" |
||
104 | fi |
||
105 | touch "$tsfile" |
||
106 | return 0 |
||
107 | ;; |
||
108 | [Nn]* ) |
||
109 | rm "$T" |
||
110 | return 1 |
||
111 | ;; |
||
112 | [Ss]* ) |
||
113 | return 1 |
||
114 | ;; |
||
115 | [Rr]* ) |
||
116 | bakmode="$mode" |
||
117 | mode="i" |
||
118 | bakforce="$force" |
||
119 | force=1 |
||
120 | process "$query" |
||
121 | RES=$? |
||
122 | mode="$bakmode" |
||
123 | force="$bakforce" |
||
124 | return $RES |
||
125 | ;; |
||
126 | [Aa]* ) |
||
127 | echo "Please enter a note you want to add or empty string if you want to cancel" |
||
128 | read note |
||
129 | |||
130 | if [ "$note" == "" ]; then |
||
131 | echo "Cancelled" |
||
132 | else |
||
133 | echo "Note added to testcase of $query" |
||
4 | daniel-mar | 134 | echo "$query: $note" >> "$ANNOTATION_FILE" |
2 | daniel-mar | 135 | fi |
136 | ;; |
||
137 | [Xx]* ) |
||
138 | return 2 |
||
139 | ;; |
||
140 | [Tt]* ) |
||
141 | # Clears the expected state |
||
142 | #rm $TESTCASE_CACHE_FILE/*/$query |
||
143 | rm "$E" |
||
144 | rm "$Q" |
||
145 | rm "$tsfile" |
||
146 | # Now retry |
||
147 | # TODO: codeduplikat vermeiden? |
||
148 | bakmode="$mode" |
||
149 | mode="i" |
||
150 | bakforce="$force" |
||
151 | force=1 |
||
152 | process "$query" |
||
153 | RES=$? |
||
154 | mode="$bakmode" |
||
155 | force="$bakforce" |
||
156 | return $RES |
||
157 | ;; |
||
158 | * ) |
||
159 | echo "Please answer with the letter written in square brackets." |
||
160 | ;; |
||
161 | esac |
||
162 | done |
||
163 | } |
||
164 | |||
165 | function question2 { |
||
166 | while true; do |
||
167 | echo "" |
||
168 | echo "Gwhois might already have been fixed. Do you want to enforce a gwhois recheck now? ([y]es, [n]o, e[x]it)" |
||
169 | read yn |
||
170 | case $yn in |
||
171 | [YyRr]* ) |
||
172 | if [ -f "$Q" ]; then |
||
173 | rm "$Q" |
||
174 | fi |
||
175 | bakmode="$mode" |
||
176 | mode="i" |
||
177 | bakforce="$force" |
||
178 | force=1 |
||
179 | process "$query" |
||
180 | RES=$? |
||
181 | mode="$bakmode" |
||
182 | force="$bakforce" |
||
183 | return $RES |
||
184 | ;; |
||
185 | [NnSs]* ) |
||
186 | return 1 |
||
187 | ;; |
||
188 | [Xx]* ) |
||
189 | return 2 |
||
190 | ;; |
||
191 | * ) |
||
192 | echo "Please answer with the letter written in square brackets." |
||
193 | ;; |
||
194 | esac |
||
195 | done |
||
196 | } |
||
197 | |||
198 | function unquote { |
||
199 | echo "$1" | sed "s/^'\(.*\)'$/\1/"; |
||
200 | } |
||
201 | |||
202 | function process { |
||
203 | if [ "$mode" != "b" ]; then |
||
204 | # TODO: diese meldung soll nicht kommen, wenn einfach nur da steht "does not need a recheck" now... |
||
205 | resetconsole |
||
206 | fi |
||
207 | |||
208 | query="$1" |
||
209 | echo "Query: $query" |
||
210 | |||
211 | E="$TESTCASE_CACHE_FILE/expected/$query" |
||
212 | Q="$TESTCASE_CACHE_FILE/problems/$query" |
||
213 | tsfile="$TESTCASE_CACHE_FILE/checktimestamps/$query" |
||
214 | |||
215 | if [ "$mode" == "u" ]; then |
||
216 | T="$Q" |
||
217 | if [ ! -f "$T" ]; then |
||
218 | echo "There is no action needed by the user" |
||
219 | return 0 |
||
220 | fi |
||
221 | else |
||
222 | T=$( mktemp --suffix='.gwhoisTC' ) |
||
223 | |||
224 | # In interactive/background mode, we will always do a webrequest when there is no problem and no expectation file without respect of the last checktime, to avoid that the status monitor will show entries with "no expectation file" when the user pressed "no", and then he would have to wait 7 days until "batch u" works again. |
||
225 | # TODO: how to do that in 1 line? |
||
226 | CALL_GWI=0 |
||
227 | if [ $force -eq 1 ]; then |
||
228 | CALL_GWI=1 |
||
229 | else |
||
230 | test \( ! -f "$Q" \) -a \( ! -f "$E" \) |
||
231 | if [ $? -eq 0 ]; then |
||
232 | CALL_GWI=1 |
||
233 | elif [ ! -f "$tsfile" ] || [ $( stat --format=%Y "$tsfile" ) -le $(( $( date +%s ) - $recheck_time )) ]; then |
||
234 | if [ -f "$Q" ] && [ $( stat --format=%Y "$Q" ) -gt $(( $( date +%s ) - $recheck_time )) ]; then |
||
235 | CALL_GWI=0 |
||
236 | else |
||
237 | CALL_GWI=1 |
||
238 | fi |
||
239 | fi |
||
240 | fi |
||
241 | if [ $CALL_GWI -eq 1 ]; then |
||
242 | echo "... calling gwhois ..." |
||
243 | # We have to use loc_gwhois to allow $trytor to work correctly. |
||
244 | # The torifiers "torify" and "usewithtor" always outputting bogus "libtorsocks" warning messages which would be saved in the output |
||
245 | # "vtor" - if applied to this script - can only filter them from STDOUT and STDERR, but not intercept this "&>" pipe |
||
246 | # So we have to use this loc_gwhois script, where we torify manually |
||
247 | # Also, it is important that we do the warning message filtering in this step, because in the final output the lines will be colored/highlighted, and therefore "vtor" cannot grep them correctly anymore. |
||
248 | # Note: regex only valid in the years 1000-1099, 1900-2099, 2900-2999 |
||
249 | # grep away volative stuff like server names or times |
||
250 | ( |
||
251 | "$DIR"/loc_gwhois "$query" \ |
||
252 | | grep -v "This query was served by " \ |
||
253 | | grep -v "(c)[12][90][0-9]\{2\}" \ |
||
254 | | grep -v "[12][90][0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" \ |
||
255 | | grep -v "\[[0-9]\{1,3\}.[0-9]\{1,3\}.[0-9]\{1,3\}.[0-9]\{1,3\} REQUEST\]" \ |
||
256 | | grep -v "RL Net \[.*\] - RL IP \[.*\]" \ |
||
257 | | grep -v "% Query time:" \ |
||
258 | | grep -v "% request from" \ |
||
259 | | grep -v "% Last update of whois database:" \ |
||
260 | | grep -v "%AM TLD whois server #" \ |
||
261 | | grep -v "Last updated on " \ |
||
262 | | grep -v "This is the Ukrainian Whois query server #" \ |
||
263 | | grep -v "Last update of whois database" \ |
||
264 | | grep -v "Query time:" \ |
||
265 | | grep -v "nsstat:" \ |
||
266 | | grep -v "nslastaa:" |
||
267 | ) &> "$T" |
||
268 | # auskommentiert wegen problem: wenn man einen recheck wegen perl errors macht und dann abbricht, ist dann der PROBLEM state gespeichert? nein, er wurde im i-mode gelöscht! |
||
269 | # -> lösung ?rm "$Q" nur im erfolgsfall |
||
270 | # if [ -f "$Q" ]; then |
||
271 | # rm "$Q" |
||
272 | # fi |
||
273 | |||
274 | # behoben? in mode=i and mode=u, should the output of a new gwhois request be saved into the problem file, if the user cancels? (to avoid a second web request?) |
||
275 | # also, when doing a recheck after a perl error, and then cancel, the problem-file will not be updated! |
||
276 | # TODO: was ist besser? |
||
277 | #if [ ! -f "$Q" ] && [ "$T" != "$Q" ]; then |
||
278 | if [ "$T" != "$Q" ]; then |
||
279 | cat "$T" > "$Q" |
||
280 | fi |
||
281 | |||
282 | |||
283 | else |
||
284 | rm "$T" |
||
285 | if [ "$mode" == "b" ]; then |
||
286 | echo "The query does not need a recheck now. Use --force to enforce it." |
||
287 | # if [ -f "$Q" ] && [ "$Q" != "$T" ]; then |
||
288 | # rm "$Q" |
||
289 | # fi |
||
290 | return 0 |
||
291 | else |
||
292 | T="$Q" |
||
293 | if [ ! -f "$T" ]; then |
||
294 | echo "There is no action needed by the user" |
||
295 | return 0 |
||
296 | fi |
||
297 | fi |
||
298 | fi |
||
299 | fi |
||
300 | |||
301 | WARNINGS=() |
||
302 | cat "$T" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null |
||
303 | if [ $? -eq 0 ]; then |
||
304 | WARNINGS+=("Perl errors found in gwhois output!") |
||
305 | fi |
||
306 | cat "$T" | head -n 1 | grep -E "^("$'\xEF\xBB\xBF'"){0,1}Process query: '$query'" > /dev/null |
||
307 | if [ $? -ne 0 ]; then |
||
308 | WARNINGS+=("The gwhois output does not begin with 'Process query'!") |
||
309 | fi |
||
310 | cat "$T" | grep "gwhois remarks: If this is a valid domainname or handle, please file a bug report." > /dev/null |
||
311 | if [ $? -eq 0 ]; then |
||
312 | WARNINGS+=("No pattern match!") |
||
313 | fi |
||
314 | if [ ${#WARNINGS[@]} -gt 0 ]; then |
||
315 | if [ "$T" != "$Q" ]; then |
||
316 | cat "$T" > "$Q" |
||
317 | rm "$T" |
||
318 | fi |
||
319 | # if [ "$mode" != "b" ]; then |
||
320 | # resetconsole |
||
321 | # fi |
||
322 | OLDIFS="$IFS" |
||
323 | IFS=$'\n' |
||
324 | for p in ${WARNINGS[@]}; do |
||
325 | echo "WARNING: $p" |
||
326 | done |
||
327 | IFS="$OLDIFS" |
||
328 | if [ "$mode" == "b" ]; then |
||
329 | echo "Saved for later analysis." |
||
330 | # TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!! |
||
331 | # touch "$tsfile" |
||
332 | return 1 |
||
333 | else |
||
334 | # Hinweis: niemals vor "question" oder "question2" ein ts aktualisieren. denn wenn man in question* einen recheck beantragt und dann abbricht, dann würde dieser aktualisierte ts bewirken, dass der testcase erst wieder in 7 tagen angezeigt wird |
||
335 | echo "" |
||
336 | cat "$Q" | showPatternHighlighted |
||
337 | question2 |
||
338 | return $? |
||
339 | fi |
||
340 | fi |
||
341 | |||
342 | if [ -f "$E" ]; then |
||
343 | D=$( date -r "$E" ) |
||
344 | echo "Compare with results of $D" |
||
345 | diff -U 0 "$E" "$T" | showPatternHighlighted |
||
346 | if [ ${PIPESTATUS[0]} -eq 0 ]; then |
||
347 | echo "OK! No differences found!" |
||
348 | rm "$T" |
||
349 | touch "$tsfile" |
||
350 | if [ -f "$Q" ] && [ "$Q" != "$T" ]; then |
||
351 | rm "$Q" |
||
352 | fi |
||
353 | return 0 |
||
354 | else |
||
355 | echo "Problem! Differences found!" |
||
356 | |||
357 | if [ "$mode" == "b" ]; then |
||
358 | echo "Saved for later analysis." |
||
359 | if [ "$T" != "$Q" ]; then |
||
360 | cat "$T" > "$Q" |
||
361 | rm "$T" |
||
362 | fi |
||
363 | |||
364 | # TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!! |
||
365 | # touch "$tsfile" |
||
366 | return 1 |
||
367 | else |
||
368 | if [ "$T" != "$Q" ]; then |
||
369 | rm "$T" |
||
370 | fi |
||
371 | |||
372 | question |
||
373 | return $? |
||
374 | fi |
||
375 | fi |
||
376 | else |
||
377 | if [ "$mode" == "b" ]; then |
||
378 | echo "This query has no expected state. Please define one." |
||
379 | echo "Saved for later analysis." |
||
380 | if [ "$T" != "$Q" ]; then |
||
381 | cat "$T" > "$Q" |
||
382 | rm "$T" |
||
383 | fi |
||
384 | # TODO: problem: this will prevent another background runner to check it again, but it will prevent the user-batch to investigate this, too?!! |
||
385 | # touch "$tsfile" |
||
386 | return 1 |
||
387 | else |
||
388 | # resetconsole |
||
389 | echo "This query has no expected state. Please define one." |
||
390 | echo "This is the current output of gwhois:" |
||
391 | |||
392 | echo "" |
||
393 | cat "$T" | showPatternHighlighted |
||
394 | echo "" |
||
395 | |||
396 | if [ "$T" != "$Q" ]; then |
||
397 | rm "$T" |
||
398 | fi |
||
399 | |||
400 | question |
||
401 | return $? |
||
402 | fi |
||
403 | fi |
||
404 | } |
||
405 | |||
406 | function HumanReadableTime { |
||
407 | local seconds=$1 |
||
408 | local days=$(($seconds/86400)) |
||
409 | seconds=$(($seconds-($days*86400) )) |
||
410 | local hours=$(($seconds/3600)) |
||
411 | seconds=$((seconds-($hours*3600) )) |
||
412 | local minutes=$(($seconds/60)) |
||
413 | seconds=$(( $seconds-($minutes*60) )) |
||
414 | |||
415 | # echo -n "${days}D ${hours}H ${minutes}M ${seconds}S" |
||
416 | if [ $days -gt 0 ]; then |
||
417 | if [ $hours -gt 12 ]; then |
||
418 | ((days++)); |
||
419 | fi |
||
420 | echo -n "${days} days" |
||
421 | elif [ $hours -gt 0 ]; then |
||
422 | if [ $minutes -gt 30 ]; then |
||
423 | ((hours++)); |
||
424 | fi |
||
425 | echo -n "${hours} hours" |
||
426 | elif [ $minutes -gt 0 ]; then |
||
427 | if [ $seconds -gt 30 ]; then |
||
428 | ((minutes++)); |
||
429 | fi |
||
430 | echo -n "${minutes} minutes" |
||
431 | else |
||
432 | echo -n "a few seconds" |
||
433 | fi |
||
434 | } |
||
435 | |||
436 | function usage { |
||
437 | . "$DIR/../../config/testcases.conf" |
||
438 | |||
439 | hf_recheck_time=$( HumanReadableTime $recheck_time ) |
||
440 | |||
441 | echo "Syntax: $0 options query" |
||
442 | echo " -h|--help" |
||
443 | echo " -m|--mode mode" |
||
444 | echo " Default: $mode" |
||
445 | echo " Mode: i = interactive (download and then show results/ask the developer if the result is OK)" |
||
446 | echo " b = background (download only and save the results for later query)" |
||
447 | echo " u = user-dialog only (only ask the developer if there are questions, e.g. which were generated by mode b)." |
||
448 | echo " -r|--rechecktime seconds" |
||
449 | echo " Default: $recheck_time = approx. $hf_recheck_time)" |
||
450 | echo " -f|--force" |
||
451 | echo " Ignores --rechecktime and forces a new gwhois request" |
||
452 | echo " Default: $force" |
||
453 | } |
||
454 | |||
455 | # --- |
||
456 | |||
457 | if [ $# -eq 0 ]; then |
||
458 | usage; |
||
459 | exit; |
||
460 | fi |
||
461 | |||
462 | if [ ! -d "$TESTCASE_CACHE_FILE/checktimestamps" ]; then |
||
3 | daniel-mar | 463 | mkdir -p "$TESTCASE_CACHE_FILE/checktimestamps" |
2 | daniel-mar | 464 | fi |
465 | |||
466 | if [ ! -d "$TESTCASE_CACHE_FILE/problems" ]; then |
||
3 | daniel-mar | 467 | mkdir -p "$TESTCASE_CACHE_FILE/problems" |
2 | daniel-mar | 468 | fi |
469 | |||
470 | if [ ! -d "$TESTCASE_CACHE_FILE/expected" ]; then |
||
3 | daniel-mar | 471 | mkdir -p "$TESTCASE_CACHE_FILE/expected" |
2 | daniel-mar | 472 | fi |
473 | |||
474 | # defaults |
||
475 | . "$DIR/../../config/testcases.conf" |
||
476 | |||
477 | PARAMS=( "$@" ); |
||
478 | optarr=( $( getopt --name "$0" --options 'hfr:m:' --long 'help,force,rechecktime:,mode:' -- "${PARAMS[@]}" 2> /dev/null ) ); |
||
479 | |||
480 | # Now process the arguments |
||
481 | i=0; |
||
482 | while true; do |
||
483 | case "${optarr[$i]}" in |
||
484 | -h|--help) |
||
485 | usage; |
||
486 | exit 0; |
||
487 | ;; |
||
488 | -f|--force) |
||
489 | force=1; |
||
490 | ((i++)); |
||
491 | ;; |
||
492 | -r|--rechecktime) |
||
493 | recheck_time=$( unquote "${optarr[$((i+1))]}" ); |
||
494 | ((i=i+2)); |
||
495 | ;; |
||
496 | -m|--mode) |
||
497 | mode=$( unquote "${optarr[$((i+1))]}" ); |
||
498 | |||
499 | if [ "$mode" != "i" ] && [ "$mode" != "b" ] && [ "$mode" != "u" ]; then |
||
500 | echo "Invalid mode '$mode'" |
||
501 | usage |
||
502 | exit 2 |
||
503 | fi |
||
504 | |||
505 | ((i=i+2)); |
||
506 | ;; |
||
507 | --) |
||
508 | ((i++)); |
||
509 | break; |
||
510 | ;; |
||
511 | *) |
||
512 | # Should never happen |
||
513 | echo "$0: Internal error while command-line-processing! Please report this error as bug." >&2; |
||
514 | exit 2; |
||
515 | ;; |
||
516 | esac |
||
517 | done |
||
518 | |||
519 | RESTARGS=${optarr[@]:i}; # i..end |
||
520 | CMD="${optarr[i]}"; |
||
521 | ARG="${optarr[@]:((i+1))}"; # (i+1)..end |
||
522 | |||
523 | |||
524 | EXITSTATUS=0 |
||
525 | for X in ${RESTARGS[@]} |
||
526 | do |
||
527 | X=$( unquote "$X" ) |
||
528 | process "$X" |
||
529 | EC=$? |
||
530 | |||
531 | if [ $EC -eq 2 ]; then |
||
532 | # user pressed x for exit |
||
533 | exit $EXITSTATUS; |
||
534 | elif [ $EC -gt $EXITSTATUS ]; then |
||
535 | # exitcode = max(exitcodes from processes) |
||
536 | EXITSTATUS=$EC |
||
537 | fi |
||
538 | |||
539 | done |
||
540 | |||
541 | exit $EXITSTATUS |