Subversion Repositories vgwhois

Compare Revisions

Regard whitespace Rev 3 → Rev 4

/trunk/maintenance/pattern-generator/.trash/generate_newgtld
16,12 → 16,10
 
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED);
 
require_once __DIR__ . '/config.inc.phps';
require_once __DIR__ . '/--/config.inc.phps';
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.phps';
require_once __DIR__ . '/../../shared/php_includes/idna_convert.class.php';
 
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
# ---
 
$iana_tld_data = null;
/trunk/maintenance/pattern-generator/config.inc.php
14,3 → 14,8
require_once __DIR__ . '/../../shared/php_includes/config_functions.inc.php';
parse_config(__DIR__ . '/../config/gwhois-pattern-update.conf');
parse_config(__DIR__ . '/../config/urls.conf');
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('DOMAINS_PATTERN_FILE', PATTERN_DIR.'/domains');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/web');
define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats');
/trunk/maintenance/pattern-generator/generate_asn
20,9 → 20,6
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/rirs.inc.php';
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
echo "ASN Preparation: Get IANA delegation file\n";
 
$iana_asn_data = explode("\n", cached_file(IANA_AS_NUMBERS, CACHE_FILE_DIR));
/trunk/maintenance/pattern-generator/generate_domains
15,10 → 15,6
require_once __DIR__ . '/config.inc.php';
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('DOMAINS_PATTERN_FILE', PATTERN_DIR.'/domains');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED);
 
$iana_tld_data = null;
/trunk/maintenance/pattern-generator/generate_ipv4
18,9 → 18,6
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/rirs.inc.php';
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
if (USE_IANA_DATA) {
echo "IPv4 Preparation: Get IANA delegation file\n";
 
/trunk/maintenance/pattern-generator/generate_ipv6
18,9 → 18,6
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/rirs.inc.php';
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
if (USE_IANA_DATA) {
echo "IPv6 Preparation: Get IANA delegation file\n";
 
/trunk/maintenance/pattern-generator/generate_tld
18,9 → 18,6
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/rirs.inc.php';
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
$out = '';
$out .= "# TLDs\n";
$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n";
/trunk/maintenance/pattern-generator/gwhois-pattern-update
11,7 → 11,7
# Version 2015-04-15
#
 
DIR=$( dirname "$0" );
DIR=$( dirname $(realpath "$0" ) )
 
. "$DIR"/../config/gwhois-pattern-update.conf
. "$DIR"/../config/urls.conf
25,28 → 25,28
echo "============================================================="
 
# RIR stats are necessary for generate_ipv4 and generate_ipv6
echo "RIR stats Synchronization ...";
echo "RIR stats Synchronization ..."
"$DIR"/sync_rir_stats
 
echo "IPv4 Generation ...";
echo "IPv4 Generation ..."
if [ $? -eq 0 ]; then
echo "IPv4 Generating ..."
"$DIR"/generate_ipv4
fi
 
echo "IPv6 Generation ...";
echo "IPv6 Generation ..."
if [ $? -eq 0 ]; then
echo "IPv6 Generating ..."
"$DIR"/generate_ipv6
fi
 
echo "ASN Generation ...";
echo "ASN Generation ..."
if [ $? -eq 0 ]; then
echo "ASN Generating ..."
"$DIR"/generate_asn
fi
 
echo "TLD Generation ...";
echo "TLD Generation ..."
if [ $? -eq 0 ]; then
echo "TLD Generating ..."
"$DIR"/generate_tld
/trunk/maintenance/pattern-generator/sync_rir_stats
17,8 → 17,6
require_once __DIR__ . '/config.inc.php';
require_once __DIR__ . '/rirs.inc.php';
 
define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats');
 
function rir_get_md5_sum($url) {
// MD5 (/var/opt/ftp/pub/apnic/stats/apnic/delegated-apnic-extended-latest) = 82c291bb5d4363a3db254853c1602777
// MD5 (delegated-lacnic-extended-latest) = 2815f0b5837d5658acf1659dff98bb52
/trunk/maintenance/qa-monitor/config.inc.php
16,3 → 16,6
parse_config(__DIR__.'/../config/urls.conf');
 
$anormale_whois=explode(' ', EXCLUDE_WHOIS);
 
define('WHOISPING_DB', __DIR__ . '/../.cache/whois-server-ping/whoisping.db');
define('DEAD_SERVER_LIST', __DIR__ . '/../config/dead-servers.list');
/trunk/maintenance/qa-monitor/run
4,12 → 4,12
#
# generic Whois - Automatic Pattern Generator: QA Monitor
#
# (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
# (c) 2012-2019 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]
#
# Distribution, usage etc. pp. regulated by the current version of GPL.
#
#
# Version 2015-04-27
# Version 2019-04-29
#
 
# TODO: strikte trennung zwischen pattern entwicklung ( = im sdk enthalten ) und lokaler gwi entwicklung ( = packages, eigene vwi services )
25,15 → 25,13
 
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED);
 
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
# ---
 
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
define('DIR', realpath(__DIR__));
 
# ---
require_once DIR . '/../../shared/php_includes/common_functions.inc.php';
 
// This is like __DIR__, but does not resolve symlinks
// (useful for sharing StatMon source codes locally by symlinking)
define('DIR', dirname($_SERVER['SCRIPT_FILENAME']));
define('CACHE_FILE_DIR', DIR . '/../.cache/web');
 
$anormale_whois = array();
require DIR . '/config.inc.php';
42,38 → 40,8
 
$global_status = 0;
 
echo "* Checking for died whois servers and check if presumed dead whois servers (= already removed from the pattern file) have resurrected...\n";
 
exec(__DIR__ . "/whois-ping/whoisping", $out, $ec);
 
$loc_good = $ec == 0;
 
if ($loc_good) {
echo "[ OK ]\n";
} else {
echo implode("\n",$out)."\n";
echo "[ FAIL ]\n";
$global_status=1;
}
 
# ---
 
echo "* Checking the testcases in background mode...\n";
 
exec(__DIR__ . "/testcases/batch --mode b", $out, $ec);
 
$loc_good = $ec == 0;
 
if ($loc_good) {
echo "[ OK ]\n";
} else {
echo implode("\n",$out)."\n";
echo "[ FAIL ]\n";
$global_status=1;
}
 
# ---
 
echo "* Check if every deleted TLD has a notice ( ".DELETED_TLD_LIST." )\n";
 
# does not exist as static file at VTS server
198,10 → 166,10
 
# ---
 
echo "* Check if there is a newer official version of gwhois which needs to be merged with the ViaThinkSoft fork ( ".DEBIAN_GWHOIS_PACKAGE_URL." )\n";
 
define('CUR_VER', gwi_getInstalledBaseVersion());
 
echo "* Check if there is a newer official version of gwhois which needs to be merged with the ViaThinkSoft fork ( ".DEBIAN_GWHOIS_PACKAGE_URL." , current base version ".CUR_VER.")\n";
 
$latest_official = getLatestGWIversion();
$latest_official = str_replace('-', '.', $latest_official); // they use the reserved '-' character in their versions (used for delimiting upstream versions)
$loc_good = true;
234,15 → 202,35
 
echo "* Check if testcases require attention\n";
 
$loc_good = true;
 
# First check tesetcases in background, if necessary
 
$out = array();
exec(__DIR__ . '/testcases/status_short', $out, $code);
$loc_good = $code == 0;
exec(DIR . "/testcases/batch --mode b", $out, $ec);
 
if ($ec > 2) {
$loc_good = false;
echo "[ !! ] testcases/batch error code $ec\n";
echo trim(implode("\n", $out))."\n";
}
 
# Now check if testcases require attention
 
$out = array();
exec(DIR . '/testcases/status_short', $out, $ec);
 
if ($ec != 0) {
$loc_good = false;
echo trim(implode("\n", $out))."\n";
}
 
# Status?
 
if ($loc_good) {
echo "[ OK ]\n";
} else {
$global_status=1;
echo trim(implode("\n", $out))."\n";
}
 
# ---
249,15 → 237,34
 
echo "* Check the status of whois-ping\n";
 
$loc_good = true;
 
# First ping the servers
 
$out = array();
exec(__DIR__ . '/whois-ping/showerrors', $out, $code);
$loc_good = $code == 0;
exec(DIR . "/whois-ping/whoisping", $out, $ec);
 
if ($ec != 0) {
$loc_good = false;
echo "[ !! ] whois-ping/whoisping error code $ec\n";
}
 
# Now show errors (but only if a time treshold is exceeded)
 
$out = array();
exec(DIR . '/whois-ping/showerrors', $out, $ec);
 
if ($ec != 0) {
$loc_good = false;
echo trim(implode("\n", $out))."\n";
}
 
# Status?
 
if ($loc_good) {
echo "[ OK ]\n";
} else {
$global_status=1;
echo trim(implode("\n", $out))."\n";
}
 
# ---
265,9 → 272,10
echo "* Check syntax of all script files (Perl, PHP)\n";
 
$out = array();
exec(__DIR__ . '/syntax/global-syntax-check', $out, $code);
$loc_good = $code == 0;
exec(DIR . '/syntax/global-syntax-check', $out, $ec);
 
$loc_good = $ec == 0;
 
if ($loc_good) {
echo "[ OK ]\n";
} else {
/trunk/maintenance/qa-monitor/testcases/allpatterns
12,9 → 12,9
# Version 2014-11-25
#
 
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php';
 
$patterns = glob(__DIR__ . '/../../main/pattern/*');
$patterns = glob(__DIR__ . '/../../../main/pattern/*');
 
#$bs = '\\b';
#$be = '\\b';
/trunk/maintenance/qa-monitor/testcases/batch
13,6 → 13,8
 
DIR=$( dirname "$0" )
 
TESTCASES_LIST="$DIR/../../config/testcases.list"
 
ARGS=()
 
shuffle() {
46,7 → 48,7
query="$f";
 
ARGS+=("$query")
done < "$DIR/../../config/testcases.list"
done < "$TESTCASES_LIST"
 
# TODO: does not work
#shuffle
/trunk/maintenance/qa-monitor/testcases/coverage
17,10 → 17,10
# => this is currently our behavior
# - Every line must be covered
 
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php';
 
$testcases = array();
$testcases_tmp = file(__DIR__ . '/../config/testcases.list');
$testcases_tmp = file(__DIR__ . '/../../config/testcases.list');
foreach ($testcases_tmp as &$tc) {
$tc = trim($tc);
if ($tc == '') continue;
28,7 → 28,7
$testcases[] = $tc;
}
 
$patterns = glob(__DIR__ . '/../../main/pattern/*');
$patterns = glob(__DIR__ . '/../../../main/pattern/*');
 
$count_total = 0;
$count_covered = 0;
/trunk/maintenance/qa-monitor/testcases/highlighter
14,8 → 14,8
 
#mb_internal_encoding('utf-8');
 
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/../../shared/php_includes/idna_convert.class.php';
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/../../../shared/php_includes/idna_convert.class.php';
 
$punycoder = new idna_convert();
 
/trunk/maintenance/qa-monitor/testcases/loc_gwhois
17,18 → 17,20
 
. "$DIR/../../config/testcases.conf"
 
GWI_EXEC="$DIR"/../../../main/gwhois
 
if [ $trytor -eq 1 ]; then
if [ -e "/usr/bin/vtor" ]; then
# We use vtor first, because it needs to cleanup temp files, which is only possible if it is not killed... Does this work?
vtor -a -r 1 -- "$DIR"/timeout.sh -t 10 -- gwhois $@
vtor -a -r 1 -- "$DIR"/timeout.sh -t 10 -- "$GWI_EXEC" $@
elif [ -e "/usr/bin/usewithtor" ]; then
"$DIR"/timeout.sh -t 10 -- usewithtor -- gwhois $@
"$DIR"/timeout.sh -t 10 -- usewithtor "$GWI_EXEC" $@
elif [ -e "/usr/bin/torify" ]; then
"$DIR"/timeout.sh -t 10 -- torify -- gwhois $@
"$DIR"/timeout.sh -t 10 -- torify "$GWI_EXEC" $@
else
echo "Warning: Cannot find any TOR proxy software. Will do the query without TOR." >&2
"$DIR"/timeout.sh -t 10 -- gwhois $@
"$DIR"/timeout.sh -t 10 -- "$GWI_EXEC" $@
fi
else
"$DIR"/timeout.sh -t 10 -- gwhois $@
"$DIR"/timeout.sh -t 10 -- "$GWI_EXEC" $@
fi
/trunk/maintenance/qa-monitor/testcases/single
17,8 → 17,7
 
DIR=$( dirname "$0" )
 
annotfile="$DIR/../../config/testcases.annot"
 
ANNOTATION_FILE="$DIR/../../config/testcases.annot"
TESTCASE_CACHE_FILE="$DIR/../../.cache/testcases"
 
# ---
57,12 → 56,12
}
 
function question {
if [ -f "$annotfile" ]; then
ANNOTS=$( cat "$annotfile" | grep -E "^$query:" )
if [ -f "$ANNOTATION_FILE" ]; then
ANNOTS=$( cat "$ANNOTATION_FILE" | grep -E "^$query:" )
if [ "$ANNOTS" != "" ]; then
echo ""
echo "********************"
echo "Notes in $annotfile :"
echo "Notes in $ANNOTATION_FILE :"
OLDIFS="$IFS"
IFS=$'\n'
for p in ${ANNOTS[@]}; do
132,7 → 131,7
echo "Cancelled"
else
echo "Note added to testcase of $query"
echo "$query: $note" >> "$annotfile"
echo "$query: $note" >> "$ANNOTATION_FILE"
fi
;;
[Xx]* )
/trunk/maintenance/qa-monitor/testcases/status
13,6 → 13,8
 
DIR=$( dirname "$0" )
 
TESTCASES_LIST="$DIR/../../config/testcases.list"
 
# ---
 
function FileAge() {
53,6 → 55,8
 
. "$DIR/../../config/testcases.conf"
 
CACHE_DIR="$DIR/../../.cache"
 
echo "Query Last activity Status"
echo "----------------------------------------------------------------------------------"
 
71,15 → 75,15
# query="$( basename $f )"
query="$f";
 
if [ ! -d "$DIR/../../.cache/testcases/expected" ]; then
mkdir -p "$DIR/../../.cache/testcases/expected"
if [ ! -d "$CACHE_DIR/testcases/expected" ]; then
mkdir -p "$CACHE_DIR/testcases/expected"
fi
expfile="$DIR/../../.cache/testcases/expected/$query"
expfile="$CACHE_DIR/testcases/expected/$query"
 
if [ ! -d "$DIR/../../.cache/testcases/checktimestamps" ]; then
mkdir -p "$DIR/../../.cache/testcases/checktimestamps"
if [ ! -d "$CACHE_DIR/testcases/checktimestamps" ]; then
mkdir -p "$CACHE_DIR/testcases/checktimestamps"
fi
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query"
tsfile="$CACHE_DIR/testcases/checktimestamps/$query"
if [ -f "$tsfile" ]; then
# lastcheck="$( date -r "$tsfile" )"
lastcheck=$( HumanReadableAge $( FileAge "$tsfile" ))
87,10 → 91,10
lastcheck="never"
fi
 
if [ ! -d "$DIR/../../.cache/testcases/problems" ]; then
mkdir -p "$DIR/../../.cache/testcases/problems"
if [ ! -d "$CACHE_DIR/testcases/problems" ]; then
mkdir -p "$CACHE_DIR/testcases/problems"
fi
errfile="$DIR/../../.cache/testcases/problems/$query"
errfile="$CACHE_DIR/testcases/problems/$query"
if [ -f "$errfile" ]; then
cat "$errfile" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null
PERLERR=$?
127,4 → 131,4
fi
 
printf "%-30s %-20s %-20s\n" "$query" "$lastcheck" "$status"
done < "$DIR/../../config/testcases.list"
done < "$TESTCASES_LIST"
/trunk/maintenance/qa-monitor/testcases/status_short
15,6 → 15,9
 
. "$DIR/../../config/testcases.conf"
 
CACHE_DIR="$DIR/../../.cache"
TESTCASES_LIST="$DIR/../../config/testcases.list"
 
count=0
 
while read f; do
31,15 → 34,15
# query="$( basename $f )"
query="$f";
 
if [ ! -d "$DIR/../../.cache/testcases/checktimestamps" ]; then
mkdir -p "$DIR/../../.cache/testcases/checktimestamps"
if [ ! -d "$CACHE_DIR/testcases/checktimestamps" ]; then
mkdir -p "$CACHE_DIR/testcases/checktimestamps"
fi
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query"
tsfile="$CACHE_DIR/testcases/checktimestamps/$query"
 
if [ ! -d "$DIR/../../.cache/testcases/problems" ]; then
mkdir -p "$DIR/../../.cache/testcases/problems"
if [ ! -d "$CACHE_DIR/testcases/problems" ]; then
mkdir -p "$CACHE_DIR/testcases/problems"
fi
errfile="$DIR/../../.cache/testcases/problems/$query"
errfile="$CACHE_DIR/testcases/problems/$query"
 
if [ ! -f "$tsfile" ]; then
((count++));
49,7 → 52,7
((count++));
fi
 
done < "$DIR/../../config/testcases.list"
done < "$TESTCASES_LIST"
 
if [ $count -eq 0 ]; then
echo "[ OK ]"
/trunk/maintenance/qa-monitor/whois-ping/config.inc.php
File deleted
/trunk/maintenance/qa-monitor/whois-ping/showerrors
16,9 → 16,12
 
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/whoisping_functions.inc.php';
require_once __DIR__ . '/config.inc.php';
require_once __DIR__ . '/../config.inc.php';
 
$db = new SQLite3(__DIR__.'/../../.cache/whois-server-ping/whoisping.db'); # TODO: in config auslagern
# ---
 
if (!is_dir(dirname(WHOISPING_DB))) mkdir(dirname(WHOISPING_DB), 0755, true);
$db = new SQLite3(WHOISPING_DB); # TODO: in config auslagern
if (!$db) die ($error);
 
gwitc_initdb($db);
40,7 → 43,7
}
 
$dead_servers = array();
$dead_servers_raw = file(__DIR__ . '/../../config/dead-servers.list');
$dead_servers_raw = file(DEAD_SERVER_LIST);
foreach ($dead_servers_raw as &$server) {
$server = trim($server);
if ($server == '') continue;
/trunk/maintenance/qa-monitor/whois-ping/whoisping
16,9 → 16,9
 
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/whoisping_functions.inc.php';
require_once __DIR__ . '/config.inc.php';
require_once __DIR__ . '/../config.inc.php';
 
define('WHOISPING_DB', __DIR__ . '/../../.cache/whois-server-ping/whoisping.db');
# ---
 
if (!is_dir(dirname(WHOISPING_DB))) mkdir(dirname(WHOISPING_DB), 0755, true);
$db = new SQLite3(WHOISPING_DB); # TODO: in config auslagern
34,7 → 34,7
}
 
$dead_servers = array();
$dead_servers_raw = file(__DIR__ . '/../../config/dead-servers.list');
$dead_servers_raw = file(DEAD_SERVER_LIST);
foreach ($dead_servers_raw as &$server) {
$server = trim($server);
if ($server == '') continue;
49,7 → 49,6
 
// We will handle them below, not here. (Otherwise the recheck timer would be reset)
if (in_array($server, $dead_servers)) {
$status=1;
echo "[ !! ] Attention! $server is in dead-servers.list and pattern. Please remove one instance!\n";
continue;
}