/trunk/maintenance/config/gwhois-pattern-update.conf |
---|
9,9 → 9,6 |
# Version 2015-04-15 |
# |
# For all |
USE_CACHE=1 |
# Only for IPv4 and IPv6 |
INCLUDE_BLOCK_RANGE_COMMENTS=0 |
/trunk/maintenance/pattern-generator/iana_functions.inc.php |
---|
File deleted |
/trunk/maintenance/pattern-generator/.trash/generate_newgtld |
---|
19,8 → 19,9 |
require_once __DIR__ . '/config.inc.phps'; |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.phps'; |
require_once __DIR__ . '/../../shared/php_includes/idna_convert.class.php'; |
require_once __DIR__ . '/iana_functions.inc.phps'; |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
# --- |
$iana_tld_data = null; |
80,7 → 81,10 |
// Only add the TLD to our pattern file if it has also been published by IANA. |
// The reason is that the newGTLD page had temporary typos many times (e.g. calogne instead of cologne) |
if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data(); |
if (!isset($iana_tld_data)) { |
$iana_tld_data = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR)); |
$iana_tld_data = array_map('trim', $iana_tld_data); |
} |
if (!in_array(strtoupper($tld), $iana_tld_data)) { |
echo "Info: Will not add $tld, since it is not yet added in IANA's registry.\n"; |
continue; |
114,7 → 118,10 |
// TODO: |
// Step 3: Check if there are IANA TLDs which are not in our pattern files yet |
if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data(); |
if (!isset($iana_tld_data)) { |
$iana_tld_data = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR)); |
$iana_tld_data = array_map('trim', $iana_tld_data); |
} |
foreach ($iana_tld_data as $tld) { |
if (does_exist($tld)) { # TODO: in allen pattern files schauen |
/trunk/maintenance/pattern-generator/generate_asn |
---|
21,18 → 21,12 |
require_once __DIR__ . '/rirs.inc.php'; |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
echo "ASN Preparation: Get IANA delegation file\n"; |
$iana_asn_url = "https://www.iana.org/assignments/as-numbers/as-numbers.txt"; |
$iana_asn_cache = IANA_CACHE_DIR . '/as-numbers.txt'; |
if ((USE_CACHE) && (file_exists($iana_asn_cache))) { |
@mkdir(dirname($iana_asn_cache), 0777, true); |
$iana_asn_data = file($iana_asn_cache, FILE_IGNORE_NEW_LINES); |
} else { |
$iana_asn_data = file($iana_asn_url, FILE_IGNORE_NEW_LINES); |
} |
$iana_asn_data = explode("\n", cached_file(IANA_AS_NUMBERS, CACHE_FILE_DIR)); |
$iana_asn_data = array_map('trim', $iana_asn_data); |
echo "ASN: Parsing...\n"; |
81,7 → 75,7 |
$out .= "# Autonomous System Numbers (ASN)\n"; |
$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
$out .= "# Source: ($version) $iana_asn_url\n"; |
$out .= "# Source: ($version) ".IANA_AS_NUMBERS."\n"; |
$out .= "\n"; |
foreach ($res as $whois => &$x) { |
/trunk/maintenance/pattern-generator/generate_domains |
---|
12,12 → 12,12 |
# Version 2015-07-13 |
# |
require_once __DIR__ . '/iana_functions.inc.php'; |
require_once __DIR__ . '/config.inc.php'; |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('DOMAINS_PATTERN_FILE', PATTERN_DIR.'/domains'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
62,9 → 62,13 |
// Step 2: |
// Search for new TLDs which are not in our pattern file |
if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data(); |
if (!isset($iana_tld_data)) { |
$iana_tld_data = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR)); |
$iana_tld_data = array_map('trim', $iana_tld_data); |
} |
foreach ($iana_tld_data as $tld) { |
if ($tld == '') continue; |
if ($tld[0] == '#') continue; |
$tld = strtolower($tld); |
/trunk/maintenance/pattern-generator/generate_ipv4 |
---|
18,21 → 18,15 |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/rirs.inc.php'; |
define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana'); |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
if (USE_IANA_DATA) { |
echo "IPv4 Preparation: Get IANA delegation file\n"; |
$iana_root_url = 'ftp://ftp.iana.org/assignments/ipv4-address-space/ipv4-address-space.txt'; |
$iana_root_cache = IANA_CACHE_DIR . '/ipv4-address-space.txt'; |
if ((USE_CACHE) && (file_exists($iana_root_cache))) { |
@mkdir(dirname($iana_root_cache), 0777, true); |
$iana_root_data = file($iana_root_cache, FILE_IGNORE_NEW_LINES); |
} else { |
$iana_root_data = file($iana_root_url, FILE_IGNORE_NEW_LINES); |
} |
$iana_root_data = explode("\n", cached_file(IANA_IPV4_ASSIGNMENTS, CACHE_FILE_DIR)); |
$iana_root_data = array_map('trim', $iana_root_data); |
$iana_root_ver = trim(str_replace('-', '', $iana_root_data[3])); |
$iana_root_data = grep($iana_root_data, 'whois.'); |
} |
48,13 → 42,7 |
echo "IPv4 $rir: Step $step - Get RIR statistics file\n"; |
$rirstat_url = $rirstat_urls[$rir]; |
$rirstat_cache = RIRSTATS_CACHE_DIR . "/$rir"; |
if ((USE_CACHE) && (file_exists($rirstat_cache))) { |
@mkdir(dirname($rirstat_cache), 0777, true); |
$rirstat_file = file($rirstat_cache); |
} else { |
$rirstat_file = file($rirstat_url); |
} |
$rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR)); |
$rirstat_file = antigrep($rirstat_file, '#'); |
74,7 → 62,7 |
$out .= "# ".strtoupper($rir)." IPv4\n"; |
$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) $iana_root_url\n"; |
if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) ".IANA_IPV4_ASSIGNMENTS."\n"; |
if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n"; |
$out .= "\n"; |
/trunk/maintenance/pattern-generator/generate_ipv6 |
---|
18,21 → 18,15 |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/rirs.inc.php'; |
define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana'); |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
if (USE_IANA_DATA) { |
echo "IPv6 Preparation: Get IANA delegation file\n"; |
$iana_root_url = 'ftp://ftp.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.txt'; |
$iana_root_cache = IANA_CACHE_DIR . '/ipv6-unicast-address-assignments.txt'; |
if ((USE_CACHE) && (file_exists($iana_root_cache))) { |
@mkdir(dirname($iana_root_cache), 0777, true); |
$iana_root_data = file($iana_root_cache, FILE_IGNORE_NEW_LINES); |
} else { |
$iana_root_data = file($iana_root_url, FILE_IGNORE_NEW_LINES); |
} |
$iana_root_data = explode("\n", cached_file(IANA_IPV6_ASSIGNMENTS, CACHE_FILE_DIR)); |
$iana_root_data = array_map('trim', $iana_root_data); |
$iana_root_ver = trim(str_replace('-', '', $iana_root_data[3])); |
$iana_root_data = grep($iana_root_data, 'whois.'); |
} |
48,13 → 42,7 |
echo "IPv6 $rir: Step $step - Get RIR statistics file\n"; |
$rirstat_url = $rirstat_urls[$rir]; |
$rirstat_cache = RIRSTATS_CACHE_DIR . "/$rir"; |
if ((USE_CACHE) && (file_exists($rirstat_cache))) { |
@mkdir(dirname($rirstat_cache), 0777, true); |
$rirstat_file = file($rirstat_cache); |
} else { |
$rirstat_file = file($rirstat_url); |
} |
$rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR)); |
$rirstat_file = antigrep($rirstat_file, '#'); |
74,7 → 62,7 |
$out .= "# ".strtoupper($rir)." IPv6\n"; |
$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) $iana_root_url\n"; |
if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) ".IANA_IPV6_ASSIGNMENTS."\n"; |
if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n"; |
$out .= "\n"; |
/trunk/maintenance/pattern-generator/generate_tld |
---|
17,7 → 17,6 |
require_once __DIR__ . '/config.inc.php'; |
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
require_once __DIR__ . '/rirs.inc.php'; |
require_once __DIR__ . '/iana_functions.inc.php'; |
define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
69,7 → 68,8 |
echo "TLD: Get IANA delegation file (".IANA_TLD_REGISTRY.")\n"; |
$iana_tld_data = get_iana_tld_data(); |
$iana_tld_data = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR)); |
$iana_tld_data = array_map('trim', $iana_tld_data); |
echo "TLD: Parsing...\n"; |
/trunk/maintenance/pattern-generator/gwhois-pattern-update |
---|
11,51 → 11,11 |
# Version 2015-04-15 |
# |
# TODO: "force" option will enforce generate even when web download was not http 200 |
DIR=$( dirname "$0" ); |
. "$DIR"/../config/gwhois-pattern-update.conf |
. "$DIR"/../config/urls.conf |
IANA_CACHE_DIR="$DIR"/../.cache/pattern-generator/iana |
function download() { |
# in: <url> <outputfile> |
# out: 0 = success |
# 1 = error |
# 2 = success, but nothing updated |
# output file will not be touched when nothing needs to be updated |
# Thanks to http://blog.yjl.im/2012/03/downloading-only-when-modified-using.html for this wonderful tutorial |
if [ -f "$2" ]; then |
HTTPCODE=$( curl -o "$2" -z "$2" -fsSL --write-out %{http_code} -- "$1" ) |
else |
HTTPCODE=$( curl -o "$2" -fsSL --write-out %{http_code} -- "$1" ) |
fi |
RES=$? |
if [ $RES -eq 0 ]; then |
touch "$2.success" |
if [[ "$HTTPCODE" == "200" ]]; then |
return 0 |
else |
return 2 |
fi |
else |
touch "$2.fail" |
return 1 |
fi |
} |
if [ $? -ne 0 ] |
then |
echo "Error when reading configuration value IANA_CACHE_DIR" > /dev/stderr |
exit 1 |
fi |
echo "=================== GWHOIS PATTERN UPDATE ===================" |
echo "This script downloads the latest RIR statistics" |
echo "and renews the pattern files of GWhoIs to provide" |
68,31 → 28,25 |
echo "RIR stats Synchronization ..."; |
"$DIR"/sync_rir_stats |
mkdir -p "$IANA_CACHE_DIR"/ |
echo "IPv4 Synchronization ..."; |
download "$IANA_IPV4_ASSIGNMENTS" "$IANA_CACHE_DIR"/ipv4-address-space.txt |
echo "IPv4 Generation ..."; |
if [ $? -eq 0 ]; then |
echo "IPv4 Generating ..." |
"$DIR"/generate_ipv4 |
fi |
echo "IPv6 Synchronization ..."; |
download "$IANA_IPV6_ASSIGNMENTS" "$IANA_CACHE_DIR"/ipv6-unicast-address-assignments.txt |
echo "IPv6 Generation ..."; |
if [ $? -eq 0 ]; then |
echo "IPv6 Generating ..." |
"$DIR"/generate_ipv6 |
fi |
echo "ASN Synchronization ..."; |
download "$IANA_AS_NUMBERS" "$IANA_CACHE_DIR"/as-numbers.txt |
echo "ASN Generation ..."; |
if [ $? -eq 0 ]; then |
echo "ASN Generating ..." |
"$DIR"/generate_asn |
fi |
echo "TLD Synchronization ..."; |
download "$IANA_TLD_REGISTRY" "$IANA_CACHE_DIR"/tlds-alpha-by-domain.txt |
echo "TLD Generation ..."; |
if [ $? -eq 0 ]; then |
echo "TLD Generating ..." |
"$DIR"/generate_tld |
/trunk/maintenance/qa-monitor/run |
---|
51,6 → 51,7 |
if ($loc_good) { |
echo "[ OK ]\n"; |
} else { |
echo implode("\n",$out)."\n"; |
echo "[ FAIL ]\n"; |
$global_status=1; |
} |
66,6 → 67,7 |
if ($loc_good) { |
echo "[ OK ]\n"; |
} else { |
echo implode("\n",$out)."\n"; |
echo "[ FAIL ]\n"; |
$global_status=1; |
} |
/trunk/maintenance/qa-monitor/testcases/single |
---|
461,15 → 461,15 |
fi |
if [ ! -d "$TESTCASE_CACHE_FILE/checktimestamps" ]; then |
mkdir "$TESTCASE_CACHE_FILE/checktimestamps" |
mkdir -p "$TESTCASE_CACHE_FILE/checktimestamps" |
fi |
if [ ! -d "$TESTCASE_CACHE_FILE/problems" ]; then |
mkdir "$TESTCASE_CACHE_FILE/problems" |
mkdir -p "$TESTCASE_CACHE_FILE/problems" |
fi |
if [ ! -d "$TESTCASE_CACHE_FILE/expected" ]; then |
mkdir "$TESTCASE_CACHE_FILE/expected" |
mkdir -p "$TESTCASE_CACHE_FILE/expected" |
fi |
# defaults |
/trunk/maintenance/qa-monitor/testcases/status |
---|
71,8 → 71,14 |
# query="$( basename $f )" |
query="$f"; |
if [ ! -d "$DIR/../../.cache/testcases/expected" ]; then |
mkdir -p "$DIR/../../.cache/testcases/expected" |
fi |
expfile="$DIR/../../.cache/testcases/expected/$query" |
if [ ! -d "$DIR/../../.cache/testcases/checktimestamps" ]; then |
mkdir -p "$DIR/../../.cache/testcases/checktimestamps" |
fi |
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query" |
if [ -f "$tsfile" ]; then |
# lastcheck="$( date -r "$tsfile" )" |
81,6 → 87,9 |
lastcheck="never" |
fi |
if [ ! -d "$DIR/../../.cache/testcases/problems" ]; then |
mkdir -p "$DIR/../../.cache/testcases/problems" |
fi |
errfile="$DIR/../../.cache/testcases/problems/$query" |
if [ -f "$errfile" ]; then |
cat "$errfile" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null |
/trunk/maintenance/qa-monitor/testcases/status_short |
---|
31,7 → 31,14 |
# query="$( basename $f )" |
query="$f"; |
if [ ! -d "$DIR/../../.cache/testcases/checktimestamps" ]; then |
mkdir -p "$DIR/../../.cache/testcases/checktimestamps" |
fi |
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query" |
if [ ! -d "$DIR/../../.cache/testcases/problems" ]; then |
mkdir -p "$DIR/../../.cache/testcases/problems" |
fi |
errfile="$DIR/../../.cache/testcases/problems/$query" |
if [ ! -f "$tsfile" ]; then |
/trunk/maintenance/qa-monitor/whois-ping/whoisping |
---|
18,7 → 18,10 |
require_once __DIR__ . '/whoisping_functions.inc.php'; |
require_once __DIR__ . '/config.inc.php'; |
$db = new SQLite3(__DIR__ . '/../../.cache/whois-server-ping/whoisping.db'); # TODO: in config auslagern |
define('WHOISPING_DB', __DIR__ . '/../../.cache/whois-server-ping/whoisping.db'); |
if (!is_dir(dirname(WHOISPING_DB))) mkdir(dirname(WHOISPING_DB), 0755, true); |
$db = new SQLite3(WHOISPING_DB); # TODO: in config auslagern |
if (!$db) die ($error); |
gwitc_initdb($db); |