Subversion Repositories vgwhois

Compare Revisions

Regard whitespace Rev 2 → Rev 3

/trunk/maintenance/config/gwhois-pattern-update.conf
9,9 → 9,6
# Version 2015-04-15
#
 
# For all
USE_CACHE=1
 
# Only for IPv4 and IPv6
INCLUDE_BLOCK_RANGE_COMMENTS=0
 
/trunk/maintenance/pattern-generator/iana_functions.inc.php
File deleted
/trunk/maintenance/pattern-generator/.trash/generate_newgtld
19,8 → 19,9
require_once __DIR__ . '/config.inc.phps';
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.phps';
require_once __DIR__ . '/../../shared/php_includes/idna_convert.class.php';
require_once __DIR__ . '/iana_functions.inc.phps';
 
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
# ---
 
$iana_tld_data = null;
80,7 → 81,10
 
// Only add the TLD to our pattern file if it has also been published by IANA.
// The reason is that the newGTLD page had temporary typos many times (e.g. calogne instead of cologne)
if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data();
if (!isset($iana_tld_data)) {
$iana_tld_data = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR));
$iana_tld_data = array_map('trim', $iana_tld_data);
}
if (!in_array(strtoupper($tld), $iana_tld_data)) {
echo "Info: Will not add $tld, since it is not yet added in IANA's registry.\n";
continue;
114,7 → 118,10
 
// TODO:
// Step 3: Check if there are IANA TLDs which are not in our pattern files yet
if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data();
if (!isset($iana_tld_data)) {
$iana_tld_data = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR));
$iana_tld_data = array_map('trim', $iana_tld_data);
}
foreach ($iana_tld_data as $tld) {
 
if (does_exist($tld)) { # TODO: in allen pattern files schauen
/trunk/maintenance/pattern-generator/generate_asn
21,18 → 21,12
require_once __DIR__ . '/rirs.inc.php';
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
echo "ASN Preparation: Get IANA delegation file\n";
 
$iana_asn_url = "https://www.iana.org/assignments/as-numbers/as-numbers.txt";
$iana_asn_cache = IANA_CACHE_DIR . '/as-numbers.txt';
if ((USE_CACHE) && (file_exists($iana_asn_cache))) {
@mkdir(dirname($iana_asn_cache), 0777, true);
$iana_asn_data = file($iana_asn_cache, FILE_IGNORE_NEW_LINES);
} else {
$iana_asn_data = file($iana_asn_url, FILE_IGNORE_NEW_LINES);
}
$iana_asn_data = explode("\n", cached_file(IANA_AS_NUMBERS, CACHE_FILE_DIR));
$iana_asn_data = array_map('trim', $iana_asn_data);
 
echo "ASN: Parsing...\n";
 
81,7 → 75,7
$out .= "# Autonomous System Numbers (ASN)\n";
$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n";
$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n";
$out .= "# Source: ($version) $iana_asn_url\n";
$out .= "# Source: ($version) ".IANA_AS_NUMBERS."\n";
$out .= "\n";
 
foreach ($res as $whois => &$x) {
/trunk/maintenance/pattern-generator/generate_domains
12,12 → 12,12
# Version 2015-07-13
#
 
require_once __DIR__ . '/iana_functions.inc.php';
require_once __DIR__ . '/config.inc.php';
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('DOMAINS_PATTERN_FILE', PATTERN_DIR.'/domains');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED);
 
62,9 → 62,13
// Step 2:
// Search for new TLDs which are not in our pattern file
 
if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data();
if (!isset($iana_tld_data)) {
$iana_tld_data = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR));
$iana_tld_data = array_map('trim', $iana_tld_data);
}
 
foreach ($iana_tld_data as $tld) {
if ($tld == '') continue;
if ($tld[0] == '#') continue;
 
$tld = strtolower($tld);
/trunk/maintenance/pattern-generator/generate_ipv4
18,21 → 18,15
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/rirs.inc.php';
 
define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana');
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
if (USE_IANA_DATA) {
echo "IPv4 Preparation: Get IANA delegation file\n";
 
$iana_root_url = 'ftp://ftp.iana.org/assignments/ipv4-address-space/ipv4-address-space.txt';
$iana_root_cache = IANA_CACHE_DIR . '/ipv4-address-space.txt';
if ((USE_CACHE) && (file_exists($iana_root_cache))) {
@mkdir(dirname($iana_root_cache), 0777, true);
$iana_root_data = file($iana_root_cache, FILE_IGNORE_NEW_LINES);
} else {
$iana_root_data = file($iana_root_url, FILE_IGNORE_NEW_LINES);
}
$iana_root_data = explode("\n", cached_file(IANA_IPV4_ASSIGNMENTS, CACHE_FILE_DIR));
$iana_root_data = array_map('trim', $iana_root_data);
 
$iana_root_ver = trim(str_replace('-', '', $iana_root_data[3]));
$iana_root_data = grep($iana_root_data, 'whois.');
}
48,13 → 42,7
echo "IPv4 $rir: Step $step - Get RIR statistics file\n";
 
$rirstat_url = $rirstat_urls[$rir];
$rirstat_cache = RIRSTATS_CACHE_DIR . "/$rir";
if ((USE_CACHE) && (file_exists($rirstat_cache))) {
@mkdir(dirname($rirstat_cache), 0777, true);
$rirstat_file = file($rirstat_cache);
} else {
$rirstat_file = file($rirstat_url);
}
$rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR));
 
$rirstat_file = antigrep($rirstat_file, '#');
 
74,7 → 62,7
$out .= "# ".strtoupper($rir)." IPv4\n";
$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n";
$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n";
if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) $iana_root_url\n";
if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) ".IANA_IPV4_ASSIGNMENTS."\n";
if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n";
$out .= "\n";
 
/trunk/maintenance/pattern-generator/generate_ipv6
18,21 → 18,15
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/rirs.inc.php';
 
define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana');
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
 
if (USE_IANA_DATA) {
echo "IPv6 Preparation: Get IANA delegation file\n";
 
$iana_root_url = 'ftp://ftp.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.txt';
$iana_root_cache = IANA_CACHE_DIR . '/ipv6-unicast-address-assignments.txt';
if ((USE_CACHE) && (file_exists($iana_root_cache))) {
@mkdir(dirname($iana_root_cache), 0777, true);
$iana_root_data = file($iana_root_cache, FILE_IGNORE_NEW_LINES);
} else {
$iana_root_data = file($iana_root_url, FILE_IGNORE_NEW_LINES);
}
$iana_root_data = explode("\n", cached_file(IANA_IPV6_ASSIGNMENTS, CACHE_FILE_DIR));
$iana_root_data = array_map('trim', $iana_root_data);
 
$iana_root_ver = trim(str_replace('-', '', $iana_root_data[3]));
$iana_root_data = grep($iana_root_data, 'whois.');
}
48,13 → 42,7
echo "IPv6 $rir: Step $step - Get RIR statistics file\n";
 
$rirstat_url = $rirstat_urls[$rir];
$rirstat_cache = RIRSTATS_CACHE_DIR . "/$rir";
if ((USE_CACHE) && (file_exists($rirstat_cache))) {
@mkdir(dirname($rirstat_cache), 0777, true);
$rirstat_file = file($rirstat_cache);
} else {
$rirstat_file = file($rirstat_url);
}
$rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR));
 
$rirstat_file = antigrep($rirstat_file, '#');
 
74,7 → 62,7
$out .= "# ".strtoupper($rir)." IPv6\n";
$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n";
$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n";
if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) $iana_root_url\n";
if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) ".IANA_IPV6_ASSIGNMENTS."\n";
if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n";
$out .= "\n";
 
/trunk/maintenance/pattern-generator/generate_tld
17,7 → 17,6
require_once __DIR__ . '/config.inc.php';
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/rirs.inc.php';
require_once __DIR__ . '/iana_functions.inc.php';
 
define('PATTERN_DIR', __DIR__ . '/../../main/pattern');
define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache');
69,7 → 68,8
 
echo "TLD: Get IANA delegation file (".IANA_TLD_REGISTRY.")\n";
 
$iana_tld_data = get_iana_tld_data();
$iana_tld_data = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR));
$iana_tld_data = array_map('trim', $iana_tld_data);
 
echo "TLD: Parsing...\n";
 
/trunk/maintenance/pattern-generator/gwhois-pattern-update
11,51 → 11,11
# Version 2015-04-15
#
 
# TODO: "force" option will enforce generate even when web download was not http 200
 
DIR=$( dirname "$0" );
 
. "$DIR"/../config/gwhois-pattern-update.conf
. "$DIR"/../config/urls.conf
 
IANA_CACHE_DIR="$DIR"/../.cache/pattern-generator/iana
 
function download() {
# in: <url> <outputfile>
# out: 0 = success
# 1 = error
# 2 = success, but nothing updated
# output file will not be touched when nothing needs to be updated
 
# Thanks to http://blog.yjl.im/2012/03/downloading-only-when-modified-using.html for this wonderful tutorial
 
if [ -f "$2" ]; then
HTTPCODE=$( curl -o "$2" -z "$2" -fsSL --write-out %{http_code} -- "$1" )
else
HTTPCODE=$( curl -o "$2" -fsSL --write-out %{http_code} -- "$1" )
fi
 
RES=$?
 
if [ $RES -eq 0 ]; then
touch "$2.success"
if [[ "$HTTPCODE" == "200" ]]; then
return 0
else
return 2
fi
else
touch "$2.fail"
return 1
fi
}
 
if [ $? -ne 0 ]
then
echo "Error when reading configuration value IANA_CACHE_DIR" > /dev/stderr
exit 1
fi
 
echo "=================== GWHOIS PATTERN UPDATE ==================="
echo "This script downloads the latest RIR statistics"
echo "and renews the pattern files of GWhoIs to provide"
68,31 → 28,25
echo "RIR stats Synchronization ...";
"$DIR"/sync_rir_stats
 
mkdir -p "$IANA_CACHE_DIR"/
 
echo "IPv4 Synchronization ...";
download "$IANA_IPV4_ASSIGNMENTS" "$IANA_CACHE_DIR"/ipv4-address-space.txt
echo "IPv4 Generation ...";
if [ $? -eq 0 ]; then
echo "IPv4 Generating ..."
"$DIR"/generate_ipv4
fi
 
echo "IPv6 Synchronization ...";
download "$IANA_IPV6_ASSIGNMENTS" "$IANA_CACHE_DIR"/ipv6-unicast-address-assignments.txt
echo "IPv6 Generation ...";
if [ $? -eq 0 ]; then
echo "IPv6 Generating ..."
"$DIR"/generate_ipv6
fi
 
echo "ASN Synchronization ...";
download "$IANA_AS_NUMBERS" "$IANA_CACHE_DIR"/as-numbers.txt
echo "ASN Generation ...";
if [ $? -eq 0 ]; then
echo "ASN Generating ..."
"$DIR"/generate_asn
fi
 
echo "TLD Synchronization ...";
download "$IANA_TLD_REGISTRY" "$IANA_CACHE_DIR"/tlds-alpha-by-domain.txt
echo "TLD Generation ...";
if [ $? -eq 0 ]; then
echo "TLD Generating ..."
"$DIR"/generate_tld
/trunk/maintenance/qa-monitor/run
51,6 → 51,7
if ($loc_good) {
echo "[ OK ]\n";
} else {
echo implode("\n",$out)."\n";
echo "[ FAIL ]\n";
$global_status=1;
}
66,6 → 67,7
if ($loc_good) {
echo "[ OK ]\n";
} else {
echo implode("\n",$out)."\n";
echo "[ FAIL ]\n";
$global_status=1;
}
/trunk/maintenance/qa-monitor/testcases/single
461,15 → 461,15
fi
 
if [ ! -d "$TESTCASE_CACHE_FILE/checktimestamps" ]; then
mkdir "$TESTCASE_CACHE_FILE/checktimestamps"
mkdir -p "$TESTCASE_CACHE_FILE/checktimestamps"
fi
 
if [ ! -d "$TESTCASE_CACHE_FILE/problems" ]; then
mkdir "$TESTCASE_CACHE_FILE/problems"
mkdir -p "$TESTCASE_CACHE_FILE/problems"
fi
 
if [ ! -d "$TESTCASE_CACHE_FILE/expected" ]; then
mkdir "$TESTCASE_CACHE_FILE/expected"
mkdir -p "$TESTCASE_CACHE_FILE/expected"
fi
 
# defaults
/trunk/maintenance/qa-monitor/testcases/status
71,8 → 71,14
# query="$( basename $f )"
query="$f";
 
if [ ! -d "$DIR/../../.cache/testcases/expected" ]; then
mkdir -p "$DIR/../../.cache/testcases/expected"
fi
expfile="$DIR/../../.cache/testcases/expected/$query"
 
if [ ! -d "$DIR/../../.cache/testcases/checktimestamps" ]; then
mkdir -p "$DIR/../../.cache/testcases/checktimestamps"
fi
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query"
if [ -f "$tsfile" ]; then
# lastcheck="$( date -r "$tsfile" )"
81,6 → 87,9
lastcheck="never"
fi
 
if [ ! -d "$DIR/../../.cache/testcases/problems" ]; then
mkdir -p "$DIR/../../.cache/testcases/problems"
fi
errfile="$DIR/../../.cache/testcases/problems/$query"
if [ -f "$errfile" ]; then
cat "$errfile" | grep -E "at /(bin|usr|etc|var)/\S+ line" > /dev/null
/trunk/maintenance/qa-monitor/testcases/status_short
31,7 → 31,14
# query="$( basename $f )"
query="$f";
 
if [ ! -d "$DIR/../../.cache/testcases/checktimestamps" ]; then
mkdir -p "$DIR/../../.cache/testcases/checktimestamps"
fi
tsfile="$DIR/../../.cache/testcases/checktimestamps/$query"
 
if [ ! -d "$DIR/../../.cache/testcases/problems" ]; then
mkdir -p "$DIR/../../.cache/testcases/problems"
fi
errfile="$DIR/../../.cache/testcases/problems/$query"
 
if [ ! -f "$tsfile" ]; then
/trunk/maintenance/qa-monitor/whois-ping/whoisping
18,7 → 18,10
require_once __DIR__ . '/whoisping_functions.inc.php';
require_once __DIR__ . '/config.inc.php';
 
$db = new SQLite3(__DIR__ . '/../../.cache/whois-server-ping/whoisping.db'); # TODO: in config auslagern
define('WHOISPING_DB', __DIR__ . '/../../.cache/whois-server-ping/whoisping.db');
 
if (!is_dir(dirname(WHOISPING_DB))) mkdir(dirname(WHOISPING_DB), 0755, true);
$db = new SQLite3(WHOISPING_DB); # TODO: in config auslagern
if (!$db) die ($error);
 
gwitc_initdb($db);