/trunk/maintenance/pattern-generator/.trash/generate_newgtld |
---|
16,12 → 16,10 |
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
require_once __DIR__ . '/config.inc.phps'; |
require_once __DIR__ . '/--/config.inc.phps'; |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.phps'; |
require_once __DIR__ . '/../../shared/php_includes/idna_convert.class.php'; |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
# --- |
$iana_tld_data = null; |
/trunk/maintenance/pattern-generator/config.inc.php |
---|
14,3 → 14,8 |
require_once __DIR__ . '/../../shared/php_includes/config_functions.inc.php'; |
parse_config(__DIR__ . '/../config/gwhois-pattern-update.conf'); |
parse_config(__DIR__ . '/../config/urls.conf'); |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('DOMAINS_PATTERN_FILE', PATTERN_DIR.'/domains'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/web'); |
define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats'); |
/trunk/maintenance/pattern-generator/generate_asn |
---|
20,9 → 20,6 |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/rirs.inc.php'; |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
echo "ASN Preparation: Get IANA delegation file\n"; |
$iana_asn_data = explode("\n", cached_file(IANA_AS_NUMBERS, CACHE_FILE_DIR)); |
/trunk/maintenance/pattern-generator/generate_domains |
---|
15,10 → 15,6 |
require_once __DIR__ . '/config.inc.php'; |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('DOMAINS_PATTERN_FILE', PATTERN_DIR.'/domains'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
$iana_tld_data = null; |
/trunk/maintenance/pattern-generator/generate_ipv4 |
---|
18,9 → 18,6 |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/rirs.inc.php'; |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
if (USE_IANA_DATA) { |
echo "IPv4 Preparation: Get IANA delegation file\n"; |
/trunk/maintenance/pattern-generator/generate_ipv6 |
---|
18,9 → 18,6 |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/rirs.inc.php'; |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
if (USE_IANA_DATA) { |
echo "IPv6 Preparation: Get IANA delegation file\n"; |
/trunk/maintenance/pattern-generator/generate_tld |
---|
18,9 → 18,6 |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/rirs.inc.php'; |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
$out = ''; |
$out .= "# TLDs\n"; |
$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
/trunk/maintenance/pattern-generator/gwhois-pattern-update |
---|
11,7 → 11,7 |
# Version 2015-04-15 |
# |
DIR=$( dirname "$0" ); |
DIR=$( dirname $(realpath "$0" ) ) |
. "$DIR"/../config/gwhois-pattern-update.conf |
. "$DIR"/../config/urls.conf |
25,28 → 25,28 |
echo "=============================================================" |
# RIR stats are necessary for generate_ipv4 and generate_ipv6 |
echo "RIR stats Synchronization ..."; |
echo "RIR stats Synchronization ..." |
"$DIR"/sync_rir_stats |
echo "IPv4 Generation ..."; |
echo "IPv4 Generation ..." |
if [ $? -eq 0 ]; then |
echo "IPv4 Generating ..." |
"$DIR"/generate_ipv4 |
fi |
echo "IPv6 Generation ..."; |
echo "IPv6 Generation ..." |
if [ $? -eq 0 ]; then |
echo "IPv6 Generating ..." |
"$DIR"/generate_ipv6 |
fi |
echo "ASN Generation ..."; |
echo "ASN Generation ..." |
if [ $? -eq 0 ]; then |
echo "ASN Generating ..." |
"$DIR"/generate_asn |
fi |
echo "TLD Generation ..."; |
echo "TLD Generation ..." |
if [ $? -eq 0 ]; then |
echo "TLD Generating ..." |
"$DIR"/generate_tld |
/trunk/maintenance/pattern-generator/sync_rir_stats |
---|
17,8 → 17,6 |
require_once __DIR__ . '/config.inc.php'; |
require_once __DIR__ . '/rirs.inc.php'; |
define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats'); |
function rir_get_md5_sum($url) { |
// MD5 (/var/opt/ftp/pub/apnic/stats/apnic/delegated-apnic-extended-latest) = 82c291bb5d4363a3db254853c1602777 |
// MD5 (delegated-lacnic-extended-latest) = 2815f0b5837d5658acf1659dff98bb52 |
/trunk/maintenance/qa-monitor/config.inc.php |
---|
16,3 → 16,6 |
parse_config(__DIR__.'/../config/urls.conf'); |
$anormale_whois=explode(' ', EXCLUDE_WHOIS); |
define('WHOISPING_DB', __DIR__ . '/../.cache/whois-server-ping/whoisping.db'); |
define('DEAD_SERVER_LIST', __DIR__ . '/../config/dead-servers.list'); |
/trunk/maintenance/qa-monitor/run |
---|
4,12 → 4,12 |
# |
# generic Whois - Automatic Pattern Generator: QA Monitor |
# |
# (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
# (c) 2012-2019 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
# |
# Distribution, usage etc. pp. regulated by the current version of GPL. |
# |
# |
# Version 2015-04-27 |
# Version 2019-04-29 |
# |
# TODO: strikte trennung zwischen pattern entwicklung ( = im sdk enthalten ) und lokaler gwi entwicklung ( = packages, eigene vwi services ) |
25,15 → 25,13 |
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
# --- |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
define('DIR', realpath(__DIR__)); |
# --- |
require_once DIR . '/../../shared/php_includes/common_functions.inc.php'; |
// This is like __DIR__, but does not resolve symlinks |
// (useful for sharing StatMon source codes locally by symlinking) |
define('DIR', dirname($_SERVER['SCRIPT_FILENAME'])); |
define('CACHE_FILE_DIR', DIR . '/../.cache/web'); |
$anormale_whois = array(); |
require DIR . '/config.inc.php'; |
42,38 → 40,8 |
$global_status = 0; |
echo "* Checking for died whois servers and check if presumed dead whois servers (= already removed from the pattern file) have resurrected...\n"; |
exec(__DIR__ . "/whois-ping/whoisping", $out, $ec); |
$loc_good = $ec == 0; |
if ($loc_good) { |
echo "[ OK ]\n"; |
} else { |
echo implode("\n",$out)."\n"; |
echo "[ FAIL ]\n"; |
$global_status=1; |
} |
# --- |
echo "* Checking the testcases in background mode...\n"; |
exec(__DIR__ . "/testcases/batch --mode b", $out, $ec); |
$loc_good = $ec == 0; |
if ($loc_good) { |
echo "[ OK ]\n"; |
} else { |
echo implode("\n",$out)."\n"; |
echo "[ FAIL ]\n"; |
$global_status=1; |
} |
# --- |
echo "* Check if every deleted TLD has a notice ( ".DELETED_TLD_LIST." )\n"; |
# does not exist as static file at VTS server |
198,10 → 166,10 |
# --- |
echo "* Check if there is a newer official version of gwhois which needs to be merged with the ViaThinkSoft fork ( ".DEBIAN_GWHOIS_PACKAGE_URL." )\n"; |
define('CUR_VER', gwi_getInstalledBaseVersion()); |
echo "* Check if there is a newer official version of gwhois which needs to be merged with the ViaThinkSoft fork ( ".DEBIAN_GWHOIS_PACKAGE_URL." , current base version ".CUR_VER.")\n"; |
$latest_official = getLatestGWIversion(); |
$latest_official = str_replace('-', '.', $latest_official); // they use the reserved '-' character in their versions (used for delimiting upstream versions) |
$loc_good = true; |
234,15 → 202,35 |
echo "* Check if testcases require attention\n"; |
$loc_good = true; |
# First check tesetcases in background, if necessary |
$out = array(); |
exec(__DIR__ . '/testcases/status_short', $out, $code); |
$loc_good = $code == 0; |
exec(DIR . "/testcases/batch --mode b", $out, $ec); |
if ($ec > 2) { |
$loc_good = false; |
echo "[ !! ] testcases/batch error code $ec\n"; |
echo trim(implode("\n", $out))."\n"; |
} |
# Now check if testcases require attention |
$out = array(); |
exec(DIR . '/testcases/status_short', $out, $ec); |
if ($ec != 0) { |
$loc_good = false; |
echo trim(implode("\n", $out))."\n"; |
} |
# Status? |
if ($loc_good) { |
echo "[ OK ]\n"; |
} else { |
$global_status=1; |
echo trim(implode("\n", $out))."\n"; |
} |
# --- |
249,15 → 237,34 |
echo "* Check the status of whois-ping\n"; |
$loc_good = true; |
# First ping the servers |
$out = array(); |
exec(__DIR__ . '/whois-ping/showerrors', $out, $code); |
$loc_good = $code == 0; |
exec(DIR . "/whois-ping/whoisping", $out, $ec); |
if ($ec != 0) { |
$loc_good = false; |
echo "[ !! ] whois-ping/whoisping error code $ec\n"; |
} |
# Now show errors (but only if a time treshold is exceeded) |
$out = array(); |
exec(DIR . '/whois-ping/showerrors', $out, $ec); |
if ($ec != 0) { |
$loc_good = false; |
echo trim(implode("\n", $out))."\n"; |
} |
# Status? |
if ($loc_good) { |
echo "[ OK ]\n"; |
} else { |
$global_status=1; |
echo trim(implode("\n", $out))."\n"; |
} |
# --- |
265,9 → 272,10 |
echo "* Check syntax of all script files (Perl, PHP)\n"; |
$out = array(); |
exec(__DIR__ . '/syntax/global-syntax-check', $out, $code); |
$loc_good = $code == 0; |
exec(DIR . '/syntax/global-syntax-check', $out, $ec); |
$loc_good = $ec == 0; |
if ($loc_good) { |
echo "[ OK ]\n"; |
} else { |
/trunk/maintenance/qa-monitor/testcases/allpatterns |
---|
12,9 → 12,9 |
# Version 2014-11-25 |
# |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php'; |
$patterns = glob(__DIR__ . '/../../main/pattern/*'); |
$patterns = glob(__DIR__ . '/../../../main/pattern/*'); |
#$bs = '\\b'; |
#$be = '\\b'; |
/trunk/maintenance/qa-monitor/testcases/batch |
---|
13,6 → 13,8 |
DIR=$( dirname "$0" ) |
TESTCASES_LIST="$DIR/../../config/testcases.list" |
ARGS=() |
shuffle() { |
46,7 → 48,7 |
query="$f"; |
ARGS+=("$query") |
done < "$DIR/../../config/testcases.list" |
done < "$TESTCASES_LIST" |
# TODO: does not work |
#shuffle |
/trunk/maintenance/qa-monitor/testcases/coverage |
---|
17,10 → 17,10 |
# => this is currently our behavior |
# - Every line must be covered |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php'; |
$testcases = array(); |
$testcases_tmp = file(__DIR__ . '/../config/testcases.list'); |
$testcases_tmp = file(__DIR__ . '/../../config/testcases.list'); |
foreach ($testcases_tmp as &$tc) { |
$tc = trim($tc); |
if ($tc == '') continue; |
28,7 → 28,7 |
$testcases[] = $tc; |
} |
$patterns = glob(__DIR__ . '/../../main/pattern/*'); |
$patterns = glob(__DIR__ . '/../../../main/pattern/*'); |
$count_total = 0; |
$count_covered = 0; |
/trunk/maintenance/qa-monitor/testcases/highlighter |
---|
14,8 → 14,8 |
#mb_internal_encoding('utf-8'); |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/../../shared/php_includes/idna_convert.class.php'; |
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/../../../shared/php_includes/idna_convert.class.php'; |
$punycoder = new idna_convert(); |
/trunk/maintenance/qa-monitor/testcases/loc_gwhois |
---|
17,18 → 17,20 |
. "$DIR/../../config/testcases.conf" |
GWI_EXEC="$DIR"/../../../main/gwhois |
if [ $trytor -eq 1 ]; then |
if [ -e "/usr/bin/vtor" ]; then |
# We use vtor first, because it needs to cleanup temp files, which is only possible if it is not killed... Does this work? |
vtor -a -r 1 -- "$DIR"/timeout.sh -t 10 -- gwhois $@ |
vtor -a -r 1 -- "$DIR"/timeout.sh -t 10 -- "$GWI_EXEC" $@ |
elif [ -e "/usr/bin/usewithtor" ]; then |
"$DIR"/timeout.sh -t 10 -- usewithtor -- gwhois $@ |
"$DIR"/timeout.sh -t 10 -- usewithtor "$GWI_EXEC" $@ |
elif [ -e "/usr/bin/torify" ]; then |
"$DIR"/timeout.sh -t 10 -- torify -- gwhois $@ |
"$DIR"/timeout.sh -t 10 -- torify "$GWI_EXEC" $@ |
else |
echo "Warning: Cannot find any TOR proxy software. Will do the query without TOR." >&2 |
"$DIR"/timeout.sh -t 10 -- gwhois $@ |
"$DIR"/timeout.sh -t 10 -- "$GWI_EXEC" $@ |
fi |
else |
"$DIR"/timeout.sh -t 10 -- gwhois $@ |
"$DIR"/timeout.sh -t 10 -- "$GWI_EXEC" $@ |
fi |
/trunk/maintenance/qa-monitor/testcases/single |
---|
17,8 → 17,7 |
DIR=$( dirname "$0" ) |
annotfile="$DIR/../../config/testcases.annot" |
ANNOTATION_FILE="$DIR/../../config/testcases.annot" |
TESTCASE_CACHE_FILE="$DIR/../../.cache/testcases" |
# --- |
57,12 → 56,12 |
} |
function question { |
if [ -f "$annotfile" ]; then |
ANNOTS=$( cat "$annotfile" | grep -E "^$query:" ) |
if [ -f "$ANNOTATION_FILE" ]; then |
ANNOTS=$( cat "$ANNOTATION_FILE" | grep -E "^$query:" ) |
if [ "$ANNOTS" != "" ]; then |
echo "" |
echo "********************" |
echo "Notes in $annotfile :" |
echo "Notes in $ANNOTATION_FILE :" |
OLDIFS="$IFS" |
IFS=$'\n' |
for p in ${ANNOTS[@]}; do |
132,7 → 131,7 |
echo "Cancelled" |
else |
echo "Note added to testcase of $query" |
echo "$query: $note" >> "$annotfile" |
echo "$query: $note" >> "$ANNOTATION_FILE" |
fi |
;; |
[Xx]* ) |
/trunk/maintenance/qa-monitor/testcases/status |
---|
13,6 → 13,8 |
DIR=$( dirname "$0" ) |
TESTCASES_LIST="$DIR/../../config/testcases.list" |
# --- |
function FileAge() { |
53,6 → 55,8 |
. "$DIR/../../config/testcases.conf" |
CACHE_DIR="$DIR/../../.cache" |
echo "Query Last activity Status" |
echo "----------------------------------------------------------------------------------" |
71,15 → 75,15 |
# query="$( basename $f )" |
query="$f"; |
if [ ! -d "$DIR/../../.cache/testcases/expected" ]; then |
mkdir -p "$DIR/../../.cache/testcases/expected" |
if [ ! -d "$CACHE_DIR/testcases/expected" ]; then |
mkdir -p "$CACHE_DIR/testcases/expected" |
fi |
expfile="$DIR/../../.cache/testcases/expected/$query" |
expfile="$CACHE_DIR/testcases/expected/$query" |
if [ ! -d "$DIR/../../.cache/testcases/checktimestamps" ]; then |
mkdir -p "$DIR/../../.cache/testcases/checktimestamps" |
if [ ! -d "$CACHE_DIR/testcases/checktimestamps" ]; then |
mkdir -p "$CACHE_DIR/testcases/checktimestamps" |
fi |
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query" |
tsfile="$CACHE_DIR/testcases/checktimestamps/$query" |
if [ -f "$tsfile" ]; then |
# lastcheck="$( date -r "$tsfile" )" |
lastcheck=$( HumanReadableAge $( FileAge "$tsfile" )) |
87,10 → 91,10 |
lastcheck="never" |
fi |
if [ ! -d "$DIR/../../.cache/testcases/problems" ]; then |
mkdir -p "$DIR/../../.cache/testcases/problems" |
if [ ! -d "$CACHE_DIR/testcases/problems" ]; then |
mkdir -p "$CACHE_DIR/testcases/problems" |
fi |
errfile="$DIR/../../.cache/testcases/problems/$query" |
errfile="$CACHE_DIR/testcases/problems/$query" |
if [ -f "$errfile" ]; then |
cat "$errfile" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null |
PERLERR=$? |
127,4 → 131,4 |
fi |
printf "%-30s %-20s %-20s\n" "$query" "$lastcheck" "$status" |
done < "$DIR/../../config/testcases.list" |
done < "$TESTCASES_LIST" |
/trunk/maintenance/qa-monitor/testcases/status_short |
---|
15,6 → 15,9 |
. "$DIR/../../config/testcases.conf" |
CACHE_DIR="$DIR/../../.cache" |
TESTCASES_LIST="$DIR/../../config/testcases.list" |
count=0 |
while read f; do |
31,15 → 34,15 |
# query="$( basename $f )" |
query="$f"; |
if [ ! -d "$DIR/../../.cache/testcases/checktimestamps" ]; then |
mkdir -p "$DIR/../../.cache/testcases/checktimestamps" |
if [ ! -d "$CACHE_DIR/testcases/checktimestamps" ]; then |
mkdir -p "$CACHE_DIR/testcases/checktimestamps" |
fi |
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query" |
tsfile="$CACHE_DIR/testcases/checktimestamps/$query" |
if [ ! -d "$DIR/../../.cache/testcases/problems" ]; then |
mkdir -p "$DIR/../../.cache/testcases/problems" |
if [ ! -d "$CACHE_DIR/testcases/problems" ]; then |
mkdir -p "$CACHE_DIR/testcases/problems" |
fi |
errfile="$DIR/../../.cache/testcases/problems/$query" |
errfile="$CACHE_DIR/testcases/problems/$query" |
if [ ! -f "$tsfile" ]; then |
((count++)); |
49,7 → 52,7 |
((count++)); |
fi |
done < "$DIR/../../config/testcases.list" |
done < "$TESTCASES_LIST" |
if [ $count -eq 0 ]; then |
echo "[ OK ]" |
/trunk/maintenance/qa-monitor/whois-ping/config.inc.php |
---|
File deleted |
/trunk/maintenance/qa-monitor/whois-ping/showerrors |
---|
16,9 → 16,12 |
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/whoisping_functions.inc.php'; |
require_once __DIR__ . '/config.inc.php'; |
require_once __DIR__ . '/../config.inc.php'; |
$db = new SQLite3(__DIR__.'/../../.cache/whois-server-ping/whoisping.db'); # TODO: in config auslagern |
# --- |
if (!is_dir(dirname(WHOISPING_DB))) mkdir(dirname(WHOISPING_DB), 0755, true); |
$db = new SQLite3(WHOISPING_DB); # TODO: in config auslagern |
if (!$db) die ($error); |
gwitc_initdb($db); |
40,7 → 43,7 |
} |
$dead_servers = array(); |
$dead_servers_raw = file(__DIR__ . '/../../config/dead-servers.list'); |
$dead_servers_raw = file(DEAD_SERVER_LIST); |
foreach ($dead_servers_raw as &$server) { |
$server = trim($server); |
if ($server == '') continue; |
/trunk/maintenance/qa-monitor/whois-ping/whoisping |
---|
16,9 → 16,9 |
require_once __DIR__ . '/../../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/whoisping_functions.inc.php'; |
require_once __DIR__ . '/config.inc.php'; |
require_once __DIR__ . '/../config.inc.php'; |
define('WHOISPING_DB', __DIR__ . '/../../.cache/whois-server-ping/whoisping.db'); |
# --- |
if (!is_dir(dirname(WHOISPING_DB))) mkdir(dirname(WHOISPING_DB), 0755, true); |
$db = new SQLite3(WHOISPING_DB); # TODO: in config auslagern |
34,7 → 34,7 |
} |
$dead_servers = array(); |
$dead_servers_raw = file(__DIR__ . '/../../config/dead-servers.list'); |
$dead_servers_raw = file(DEAD_SERVER_LIST); |
foreach ($dead_servers_raw as &$server) { |
$server = trim($server); |
if ($server == '') continue; |
49,7 → 49,6 |
// We will handle them below, not here. (Otherwise the recheck timer would be reset) |
if (in_array($server, $dead_servers)) { |
$status=1; |
echo "[ !! ] Attention! $server is in dead-servers.list and pattern. Please remove one instance!\n"; |
continue; |
} |