| /trunk/maintenance/pattern-generator/.trash/generate_newgtld |
|---|
| 0,0 → 1,319 |
| #!/usr/bin/php |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator: New gTLDs |
| # |
| # (c) 2012-2014 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2014-10-30 |
| # |
| # TODO: ":notice||Whois server unknown (2014-03-15)" in pattern_newgtld automatisch umwandeln in :whois| wenn etwas bei IANA vorliegt. |
| error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
| require_once __DIR__ . '/config.inc.phps'; |
| require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.phps'; |
| require_once __DIR__ . '/../../shared/php_includes/idna_convert.class.php'; |
| require_once __DIR__ . '/iana_functions.inc.phps'; |
| # --- |
| $iana_tld_data = null; |
| // Step 1: |
| // Check if in the meantime TLDs without an official whois server were updated to have one |
| // Attention/TODO: A change of the whois name still needs manual intervention! |
| $newgtld_cont_original = file_get_contents(NEWGTLD_PATTERN_FILE); |
| $newgtld_cont_new = preg_replace_callback( |
| '@# TODO: Entry generated automatically\. Needs manual check\.\n:notice\|\|Whois server unknown \((.*)\)\n\.(.*)\$\n@imU', |
| function ($treffer) { |
| $in_all = $treffer[0]; |
| $in_ts = $treffer[1]; |
| $in_tld = $treffer[2]; |
| $days_passed = (time()-strtotime($treffer[1]))/(60*60*24); |
| if ($days_passed < NEWGTLD_RECHECK_MISSING_WHOIS_SERVERS) { |
| return $in_all; // leave everything unchanged |
| } |
| $whois_serv = find_rootzone_whois_server($in_tld); |
| if (!$whois_serv) { |
| // Nothing found. Just update last check date. |
| return str_replace($in_ts, date('Y-m-d'), $in_all); |
| } else { |
| // Update the entry |
| return ":whois|$whois_serv\n.${in_tld}\$\n"; |
| } |
| }, |
| $newgtld_cont_original |
| ); |
| if ($newgtld_cont_original != $newgtld_cont_new) { |
| file_put_contents(NEWGTLD_PATTERN_FILE, $newgtld_cont_new); |
| gwi_update_newgtld_patternfile(); |
| } |
| // Step 2: |
| // Search for new gTLDs which are not in our pattern file |
| $newgtld_data = gwi_newgtld_get_all_delegated_strings(); |
| foreach ($newgtld_data as $data) { |
| $date = $data[0]; |
| $string = $data[1]; |
| $tld = $data[2]; |
| $tld_uc = $data[3]; |
| $explanation = $data[4]; |
| // Is it already in our pattern file? |
| if (does_exist($tld)) { |
| # echo "Info: $tld is already in pattern.\n"; |
| continue; |
| } |
| // Only add the TLD to our pattern file if it has also been published by IANA. |
| // The reason is that the newGTLD page had temporary typos many times (e.g. calogne instead of cologne) |
| if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data(); |
| if (!in_array(strtoupper($tld), $iana_tld_data)) { |
| echo "Info: Will not add $tld, since it is not yet added in IANA's registry.\n"; |
| continue; |
| } |
| $to_append = "\n"; |
| $to_append .= "# Delegated on $date\n"; |
| if ($explanation) { |
| $to_append .= "# $explanation\n"; |
| } |
| $whois_serv = find_rootzone_whois_server($tld); |
| if ($whois_serv) { |
| $to_append .= ":whois|$whois_serv\n"; |
| } else { |
| $to_append .= "# TODO: Entry generated automatically. Needs manual check.\n"; |
| $to_append .= ":notice||Whois server unknown (".date('Y-m-d').")\n"; |
| } |
| $to_append .= "\\.$tld$\n"; |
| file_put_contents(NEWGTLD_PATTERN_FILE, $to_append, FILE_APPEND); |
| gwi_update_newgtld_patternfile(); |
| } |
| // TODO: |
| // Step 3: Check if there are IANA TLDs which are not in our pattern files yet |
| if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data(); |
| foreach ($iana_tld_data as $tld) { |
| if (does_exist($tld)) { # TODO: in allen pattern files schauen |
| continue; |
| } |
| # echo "Does not exist: $tld\n"; |
| continue; |
| $newgtld_res = count_newgtld_applications($tld); |
| if ($newgtld_res === false) { |
| // TODO: ignore? |
| } else if ($newgtld_res > 0) { |
| // TODO: add to newgtld pattern file |
| } else { |
| // TODO: add to normal pattern file |
| } |
| } |
| # ------------------------------------------------------ |
| function gwi_update_newgtld_patternfile() { |
| $now = date('Ymd'); |
| $pcont_original = file_get_contents(NEWGTLD_PATTERN_FILE); |
| $pcont = $pcont_original; |
| $count = 0; |
| $pcont = preg_replace("@#: version (\\S+)@i", "#: version $now", $pcont, 1, $count); |
| if ($count == 0) { |
| // Add header |
| $pcont = "#: version $now\n". |
| "# New gTLD\n". |
| "# see: http://newgtlds.icann.org/en/program-status/delegated-strings\n". |
| "# This file can be updated by ".__DIR__."/generate_newgtld (only additions of new \"New gTLDs\")\n". |
| "# --------------------------------------------------------------------\n". |
| "\n".$pcont; |
| } |
| if ($pcont != $pcont_original) { |
| file_put_contents(NEWGTLD_PATTERN_FILE, $pcont); |
| } |
| } |
| function find_rootzone_whois_server($tld) { |
| $whois_serv = iana_get_rootzone_whois_server($tld); |
| // Try to find "secret whois servers" |
| if (TRY_FINDING_HIDDEN_WHOIS_SERVERS) { |
| // TODO: also try out to use the URL of the homepage (in IANAs root DB) |
| if (!$whois_serv) { |
| $check_server = "whois.nic.$tld"; |
| if (gwitc_is_port_open($check_server, 43)) { |
| $whois_serv = $check_server; |
| } |
| } |
| if (!$whois_serv) { |
| $check_server = "whois.$tld"; |
| if (gwitc_is_port_open($check_server, 43)) { |
| $whois_serv = $check_server; |
| } |
| } |
| if (!$whois_serv) { |
| $check_server = "$tld"; |
| if (gwitc_is_port_open($check_server, 43)) { |
| $whois_serv = $check_server; |
| } |
| } |
| } |
| return $whois_serv; |
| } |
| function iana_get_rootzone_whois_server($tld) { |
| $tld = strtolower($tld); |
| $cont = QueryWhoisServer('whois.iana.org', $tld); |
| if (!preg_match('@whois:\\s*(\\S+)@i', $cont, $m)) return false; |
| return $m[1]; |
| } |
| function does_exist($tld) { |
| $cont = file_get_contents(NEWGTLD_PATTERN_FILE); |
| $tld = strtolower($tld); |
| $cont = strtolower($cont); |
| return (strpos($cont, "\n\\.$tld\$\n") !== false); |
| } |
| function gwi_newgtld_get_all_delegated_strings() { |
| $cont = file_get_contents('http://newgtlds.icann.org/en/program-status/delegated-strings'); |
| // Convert Unicode stuff |
| $cont = str_replace('xn'.unichr(0x2013), 'xn--', $cont); |
| $cont = str_replace('xn'.unichr(0x2015), 'xn--', $cont); // used in Samsung TLD |
| $cont = str_replace(unichr(0x2013), '-', $cont); // used in most explanations |
| $cont = str_replace(unichr(0x2015), '-', $cont); |
| $cont = str_replace(unichr(0x00fc), 'ue', $cont); // German umlaut ue (used in .koeln) |
| $cont = utf8_decode($cont); |
| // Do some minor corrections |
| $cont = str_replace('game (s)', 'game(s)', $cont); |
| $cont = explode('STRING</th>', $cont, 2); |
| $cont = $cont[1]; |
| preg_match_all('@<tr>\s*<td[^>]*>(.*)</td>\s*<td[^>]*>(.*)</td>\s*</tr>@ismU', $cont, $m, PREG_SET_ORDER); |
| $m = array_reverse($m); |
| $out = array(); |
| foreach ($m as $data) { |
| $date = html_entity_decode(strip_tags($data[1])); |
| $string = html_entity_decode(strip_tags($data[2])); |
| $string = str_replace('(', ' (', $string); |
| while (strpos($string, ' ') !== false) $string = str_replace(' ', ' ', $string); |
| # Fixing some misplaced white spaces |
| $string = preg_replace('@\.\s+@m', '.', $string); |
| $string = preg_replace('@\(\s+@m', '(', $string); |
| $string = preg_replace('@\s+\)@m', ')', $string); |
| $ary = explode(' ', $string, 3); |
| if (count($ary) > 1) { |
| $tld_uc = trim($ary[0]); // Unicode TLD |
| $tld = trim($ary[1]); // Punycode TLD |
| $tld = substr($tld, 1, strlen($tld)-2); |
| $explanation = trim($ary[2]); |
| $explanation = substr($explanation, 1); // remove "-" |
| $explanation = trim($explanation); |
| } else { |
| $tld = trim(strtolower($string)); |
| $tld_uc = $tld; |
| $explanation = ''; |
| } |
| $out[] = array($date, $string, $tld, $tld_uc, $explanation); |
| } |
| return $out; |
| } |
| /** |
| * Return unicode char by its code |
| * |
| * @param int $u |
| * @return char |
| * @source http://www.php.net/manual/de/function.chr.php#88611 |
| */ |
| function unichr($u) { |
| return mb_convert_encoding('&#' . intval($u) . ';', 'UTF-8', 'HTML-ENTITIES'); |
| } |
| function count_newgtld_applications($string) { |
| #global $punycoder; |
| #if (is_null($punycoder)) |
| $punycoder = new idna_convert(); |
| $ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0'; |
| if (($x = $punycoder->decode(strtolower($string))) !== false) $string = $x; |
| $out = array(); |
| exec('curl -i -s https://gtldresult.icann.org/application-result/applicationstatus/viewstatus -H "Host: gtldresult.icann.org" -H '.escapeshellarg('User-Agent: '.$ua), $out, $code); |
| if ($code != 0) return false; |
| $html = implode("\n", $out); |
| if (!preg_match('@JSESSIONID=(.+);@ismU', $html, $m)) return false; |
| $jsessionid = $m[1]; |
| if (!preg_match('@<input value="([^"]+)" name="t:formdata"@ismU', $html, $m)) return false; |
| $formdata = $m[1]; |
| $formdata = str_replace('+', '%2B', $formdata); |
| $formdata = str_replace('/', '%2F', $formdata); |
| $out = array(); |
| exec('curl -s "https://gtldresult.icann.org/application-result/applicationstatus/viewstatus.applicationstatusform" -H "Host: gtldresult.icann.org" -H '.escapeshellarg('User-Agent: '.$ua).' -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" -H "Accept-Language: de,en-US;q=0.7,en;q=0.3" -H "Referer: https://gtldresult.icann.org/application-result/applicationstatus/viewstatus" -H '.escapeshellarg('Cookie: JSESSIONID='.$jsessionid).' -H "Connection: keep-alive" --data '.escapeshellarg('t%3Aformdata='.$formdata.'&t%3Asubmit=%5B%22searchButton%22%2C%22searchButton%22%5D&searchField='.$string.'&searchButton=Search&status=&updates=&objections=&gacew=&similar=&pic='), $out, $code); |
| if ($code != 0) return false; |
| $out = array(); |
| exec('curl -s "https://gtldresult.icann.org/application-result/applicationstatus/viewstatus" -H "Host: gtldresult.icann.org" -H '.escapeshellarg('User-Agent: '.$ua).' -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" -H "Accept-Language: de,en-US;q=0.7,en;q=0.3" -H "Referer: https://gtldresult.icann.org/application-result/applicationstatus/viewstatus" -H '.escapeshellarg('Cookie: JSESSIONID='.$jsessionid).' -H "DNT: 1" -H "Connection: keep-alive"', $out, $code); |
| if ($code != 0) return false; |
| $html = implode("\n", $out); |
| preg_match_all('@href="/application-result/applicationstatus/viewstatus:viewapplicationdetails/(\d+)">'.preg_quote($string, '@').'</a>@ismU', $html, $m); |
| # DEBUG |
| echo "$string = ".count($m[1])."\n"; |
| # return (count($m[1]) > 0); |
| return count($m[1]); |
| } |
| #assert(count_newgtld_applications('shopping') == 2); |
| Property changes: |
| Added: svn:executable |
| +* |
| \ No newline at end of property |
| /trunk/maintenance/pattern-generator/.trash/test-ipv4_functions.inc.php2.2-2.4 |
|---|
| 0,0 → 1,684 |
| <?php |
| # |
| # IPv4 Functions for PHP |
| # |
| # (c) 2012 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2012-12-17 |
| # |
| // TODO: oop |
| // Very small self-test: |
| /* |
| function ipv4_selftest() { |
| $iv_b = ipv4_complete('1.2'); |
| $iv_m = 20; |
| $r = ipv4_cidr2range($iv_b, $iv_m); |
| echo "$iv_b/$iv_m => $r[0] - $r[1]\n"; |
| $rev = ipv4_range2cidr($r[0], $r[1]); |
| $rev = implode("\n", $rev); |
| echo "$r[0] - $r[1] => $rev ["; |
| $ok = $rev == "$iv_b/$iv_m"; |
| echo $ok ? 'OK' : 'Mismatch'; |
| echo "]\n"; |
| echo "In-CIDR-Test: "; |
| echo ipv4_in_cidr("$iv_b/$iv_m", "$iv_b/$iv_m") ? 'OK' : 'Fail'; |
| echo "\n"; |
| } |
| ipv4_selftest(); |
| */ |
| function ipv4_cidr2range($baseip_or_cidr, $subnet='') { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.1 |
| # This function converts an CIDR notation <baseip>/<subnet> into an IPv4 address block array($low_ip, $high_ip) |
| if (strpos($baseip_or_cidr, '/') !== false) { |
| $tmp = explode('/', $baseip_or_cidr, 2); |
| $baseip_or_cidr = $tmp[0]; |
| $subnet = $tmp[1]; |
| unset($tmp); |
| } |
| if (($subnet < 0) || ($subnet > 32)) return false; |
| $maxint32 = 0xFFFFFFFF; |
| $netmask = $maxint32 << (32-$subnet); |
| $netmask = $netmask & $maxint32; // crop to 32 bits |
| $wildcard = $maxint32 ^ $netmask; // ~$netmask; |
| $x = ipv4_incomplete_ip2long($baseip_or_cidr) & $netmask; |
| $nums = $wildcard; |
| $low = long2ip($x); |
| $high = long2ip($x + $nums); |
| return array($low, $high); |
| } |
| function ipv4_range2cidr($baseip, $topip, $shortening=false) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| # This function converts an IPv4 address block into valid CIDR blocks (There may be multiple blocks!) |
| $out = array(); |
| if (ipv4_cmp($baseip, $topip) > 0) return false; |
| while (ipv4_incomplete_ip2long($baseip)-1 != ipv4_incomplete_ip2long($topip)) { |
| $i = -1; |
| do { |
| $i++; |
| $range = ipv4_cidr2range($baseip, $i); |
| $l = $range[0]; |
| $t = $range[1]; |
| } while ((ipv4_cmp($l, $baseip) != 0) || (ipv4_cmp($t, $topip) > 0)); |
| # Shortening: Stroke ".0" at the end |
| if ($shortening) $baseip = ipv4_shortening($baseip); |
| $out[] = "$baseip/$i"; |
| $baseip = ipv4_add($t, 1); |
| } |
| return $out; |
| } |
| function ipv4_shortening($ip) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| return preg_replace("|(\\.0{1,3}){0,3}\$|ismU", '', $ip); |
| } |
| function ipv4_add($baseip, $num) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| return long2ip(ipv4_incomplete_ip2long($baseip) + $num); |
| } |
| function ipv4_sub($baseip, $num) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| return long2ip(ipv4_incomplete_ip2long($baseip) - $num); |
| } |
| function ipv4_cmp($a, $b) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| $a = ipv4_incomplete_ip2long($a); |
| $b = ipv4_incomplete_ip2long($b); |
| if ($a == $b) return 0; |
| if ($a < $b) return -1; |
| if ($a > $b) return 1; |
| } |
| function ipv4_in_cidr($haystack, $needle) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.1 |
| $x = explode('/', $haystack); |
| $ha = ipv4_cidr2range($x[0], $x[1]); |
| $x = explode('/', $needle); |
| if (!isset($x[1])) $x[1] = '32'; // single IP |
| $ne = ipv4_cidr2range($x[0], $x[1]); |
| $ha_low = ipv4_incomplete_ip2long($ha[0]); |
| $ha_hig = ipv4_incomplete_ip2long($ha[1]); |
| $ne_low = ipv4_incomplete_ip2long($ne[0]); |
| $ne_hig = ipv4_incomplete_ip2long($ne[1]); |
| # HA: low[ ]high |
| # NE: low[ ]high |
| return ($ne_low >= $ha_low) && ($ne_hig <= $ha_hig); |
| } |
| function ipv4_complete($short_form) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| $short_form = trim($short_form); |
| if ($short_form == '') return '0.0.0.0'; |
| $c = substr_count($short_form, '.'); |
| if ($c > 3) return false; |
| if ($c == 3) return $short_form; |
| $c = substr_count($short_form, '.'); |
| $short_form .= str_repeat('.0', 3-$c); |
| return $short_form; |
| } |
| function ipv4_incomplete_ip2long($ip) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.1 |
| # return ip2long(ipv4_complete($ip)); |
| return ip2long(ipv4_normalize($ip)); |
| } |
| function ipv4_merge_address_blocks($data, $debug = false, $shortening = false) { |
| # (C) 2012 ViaThinkSoft |
| # Version 2.1 |
| if ($debug !== false) $STARTZEIT = time(); |
| // 1. Convert IPs to numbers |
| $cmp_ary = array(); |
| foreach ($data as $a => &$b) { |
| $a = ipv4_incomplete_ip2long($a); |
| $b = ipv4_incomplete_ip2long($b); |
| $cmp_ary[$a] = $b; |
| unset($a); |
| unset($b); |
| } |
| // 2. Merge the blocks in an intelligent way (and remove redundant blocks) |
| # Merge overlapping blocks |
| # [ ] |
| # [ ] -> [ ] |
| # Merge neighbor blocks |
| # [ ][ ] -> [ ] |
| # Remove redundant blocks |
| # [ ] -> [ ] |
| # [ ] |
| /* |
| $merge_count = 0; |
| $redundant_deleted_count = 0; |
| $round_count = 0; |
| do { |
| if ($debug !== false) { |
| $LAUFZEIT = time() - $STARTZEIT; |
| echo $debug."Merging... $round_count rounds; merged $merge_count blocks; deleted $redundant_deleted_count redundant blocks; time: $LAUFZEIT seconds\r"; |
| } |
| $round_count++; |
| $clean = true; |
| foreach ($cmp_ary as $a => &$b) { |
| foreach ($cmp_ary as $x => &$y) { |
| if ($x<=$a) continue; |
| if ($x>$b+1) continue; // <-- grenzfall!!! falsch!!! |
| // Merge |
| $clean = false; |
| if ($y>=$b) { // <-- y>b? |
| $merge_count++; |
| $b = $y; |
| unset($cmp_ary[$x]); // <!-- race condition? |
| break; // increases performance dramatically |
| } else { |
| $redundant_deleted_count++; |
| unset($cmp_ary[$x]); // <-- race condition? |
| } |
| } |
| } |
| } while (!$clean); |
| if ($debug !== false) { |
| $LAUFZEIT = time() - $STARTZEIT; |
| echo $debug."Merge completed. $round_count rounds; merged $merge_count blocks; deleted $redundant_deleted_count redundant blocks; time: $LAUFZEIT seconds\n"; |
| } |
| // 3. Sort |
| ksort($cmp_ary); |
| */ |
| // IMPORTANT! $cmp_ary[x]=y MUST HAVE x<=y ! |
| // To avoid two nested foreach() loops, we sort the array by source, so we can |
| // easily only scan an area using a for-loop. |
| ksort($cmp_ary); |
| if ($debug !== false) { |
| $LAUFZEIT = time() - $STARTZEIT; |
| echo $debug."Merge - Step 1; time: $LAUFZEIT seconds\n"; |
| } |
| # ================================================= |
| # Rule 1: Expand |
| # --- Minimum case: |
| # Before: |
| # [a b] |
| # [x y] |
| # After: |
| # [a b] (set b:=y) |
| # [x y] (obsolete, will be deleted in step 2) |
| # --- Maximum case: |
| # Before: |
| # [a b] |
| # [x y] |
| # After: |
| # [a b] (set b:=y) |
| # [x y] (obsolete, will be deleted in step 2) |
| # --- Rule: |
| # a<=x<=b+1 && y>b ==> Blocks a..b and x..y merge to single block a..y |
| # ================================================= |
| $keys = array_keys($cmp_ary); |
| $idx_a = 0; |
| foreach ($cmp_ary as $a => &$b) { |
| // $x=$a not possible because it's the array key and is therefore unique! |
| for ($i=++$idx_a; isset($keys[$i]) && ($keys[$i]/*x*/<=$b+1); $i++) { |
| $b = $cmp_ary[$keys[$i]]; // $y |
| break; // test 2.3 |
| } |
| /* |
| for ($x=$a+1; $x<=$b+1; $x++) { |
| if (!isset($cmp_ary[$x])) continue; |
| // Merge! Set b:=y . |
| // Block [x..y] will be deleted in step 2 |
| // because it is then a subset of [a..b=y] |
| $b = $cmp_ary[$x]; // $y |
| } |
| */ |
| } |
| if ($debug !== false) { |
| $LAUFZEIT = time() - $STARTZEIT; |
| echo $debug."Merge - Step 2; time: $LAUFZEIT seconds\n"; |
| } |
| # ================================================= |
| # Step 2: Clean up |
| # --- Minimum case: |
| # Before: |
| # [a b] |
| # [x=y] |
| # After: |
| # [a b] |
| # --- Maximum case: |
| # Before: |
| # [a b] |
| # [x y] |
| # After: |
| # [a b] |
| # --- Rule: |
| # a<=x<=y<=b ==> Block x..y is not neccessary |
| # The expression y<=b is ALWAYS true, because Step 1 would |
| # have extended b=y if y>b would have been the case. |
| # So the formula is: |
| # a<=x<=b ==> Block x..y is not neccessary |
| # ================================================= |
| $out_ary = array(); |
| $prev = -1; |
| foreach ($cmp_ary as $a => &$b) { |
| if ($a <= $prev) continue; |
| $out_ary[$a] = $b; |
| $prev = $b; |
| } |
| $cmp_ary = $out_ary; |
| if ($debug !== false) { |
| $LAUFZEIT = time() - $STARTZEIT; |
| echo $debug."Merge - Finished; time: $LAUFZEIT seconds\n"; |
| } |
| // 4. Convert back to IPs |
| $out_ary = array(); |
| foreach ($cmp_ary as $a => &$b) { |
| $a = long2ip($a); |
| $b = long2ip($b); |
| if ($shortening) { |
| $a = ipv4_shortening($a); |
| $b = ipv4_shortening($b); |
| } |
| $out_ary[$a] = $b; |
| } |
| return $out_ary; |
| } |
| function ipv4_merge_arrays($data_a, $data_b) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.2 |
| $normalized_data_a = array(); |
| foreach ($data_a as $from => &$to) { |
| $normalized_data_a[ipv4_normalize($from)] = ipv4_normalize($to); |
| } |
| $normalized_data_b = array(); |
| foreach ($data_b as $from => &$to) { |
| $normalized_data_b[ipv4_normalize($from)] = ipv4_normalize($to); |
| } |
| $data = array(); |
| foreach ($normalized_data_a as $from => &$to) { |
| if (isset($normalized_data_b[$from])) { |
| $data[$from] = ipv4_max($to, $normalized_data_b[$from]); |
| } else { |
| $data[$from] = $to; |
| } |
| } |
| foreach ($normalized_data_b as $from => &$to) { |
| if (!isset($normalized_data_a[$from])) { |
| $data[$from] = $to; |
| } |
| } |
| return $data; |
| } |
| function ipv4_valid($ip) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| # return ipv4_incomplete_ip2long($ip) !== false; |
| return ip2long($ip) !== false; |
| } |
| function ipv4_normalize($ip) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| # Example: |
| # 100.010.001.000 -> 100.10.1.0 |
| $ip = ipv4_complete($ip); |
| # ip2long buggy: 001.0.0.0 wird nicht akzeptiert |
| $cry = explode('.', $ip); |
| $cry[0] = preg_replace('@^0+@', '', $cry[0]); if ($cry[0] == '') $cry[0] = '0'; |
| $cry[1] = preg_replace('@^0+@', '', $cry[1]); if ($cry[1] == '') $cry[1] = '0'; |
| $cry[2] = preg_replace('@^0+@', '', $cry[2]); if ($cry[2] == '') $cry[2] = '0'; |
| $cry[3] = preg_replace('@^0+@', '', $cry[3]); if ($cry[3] == '') $cry[3] = '0'; |
| $ip = implode('.', $cry); |
| return $ip; |
| } |
| function ipv4_expand($ip) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| # Example: |
| # 100.10.1.0 -> 100.010.001.000 |
| $ip = ipv4_complete($ip); |
| $cry = explode('.', $ip); |
| $cry[0] = str_pad($cry[0], 3, '0', STR_PAD_LEFT); |
| $cry[1] = str_pad($cry[1], 3, '0', STR_PAD_LEFT); |
| $cry[2] = str_pad($cry[2], 3, '0', STR_PAD_LEFT); |
| $cry[3] = str_pad($cry[3], 3, '0', STR_PAD_LEFT); |
| return implode('.', $cry); |
| } |
| function ipv4_min($ip_a, $ip_b) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| if (ipv4_cmp($ip_a, $ip_b) == -1) { |
| return $ip_a; |
| } else { |
| return $ip_b; |
| } |
| } |
| function ipv4_max($ip_a, $ip_b) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| if (ipv4_cmp($ip_a, $ip_b) == 1) { |
| return $ip_a; |
| } else { |
| return $ip_b; |
| } |
| } |
| function ipv4_ipcount($data) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| $cnt = 0; |
| foreach ($data as $from => &$to) { |
| $cnt += ipv4_incomplete_ip2long($to) - ipv4_incomplete_ip2long($from); |
| } |
| return $cnt; |
| } |
| function ipv4_read_file($file) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| $data = array(); |
| $lines = file($file); |
| foreach ($lines as &$line) { |
| $rng = ipv4_line2range($line); |
| $data[$rng[0]] = $rng[1]; |
| } |
| return $data; |
| } |
| function ipv4_line2range($line) { |
| # (C) 2012 ViaThinkSoft |
| # Version 1.0 |
| $line = trim($line); |
| if (strpos($line, '/') !== false) { |
| $rng = ipv4_cidr2range($line); |
| } else { |
| $rng = explode('-', $line); |
| $rng[0] = trim($rng[0]); |
| $rng[1] = trim($rng[1]); |
| $rng[0] = ipv4_normalize($rng[0]); |
| if (!isset($rng[1])) $rng[1] = $rng[0]; |
| $rng[1] = ipv4_normalize($rng[1]); |
| } |
| return $rng; |
| } |
| # --- New 16,12,12 |
| define('IPV4_BITS', 32); |
| function ipv4_distance($ipOrCIDR_Searchterm, $ipOrCIDR_Candidate) { |
| $ary = ipv4_cidr_split($ipOrCIDR_Searchterm); |
| $ip = $ary[0]; |
| if (filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false) { |
| return false; |
| } |
| $ary = ipv4_cidr_split($ipOrCIDR_Candidate); |
| $ip = $ary[0]; |
| $cidr_bits = $ary[1]; |
| if ($cidr_bits > IPV4_BITS) return false; // throw new Exception('CIDR bits > '.IPV4_BITS); |
| if (filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false) { |
| return false; |
| } |
| $x = ipv4_trackdown($ipOrCIDR_Searchterm); |
| if (ipv4_in_cidr($x[0], $ip.'/'.$cidr_bits)) { |
| $ary = ipv4_cidr_split($x[0]); |
| $cidr_bits2 = $ary[1]; |
| if ($cidr_bits2 > IPV4_BITS) return false; // throw new Exception('CIDR bits > '.IPV4_BITS); |
| return $cidr_bits2-$cidr_bits; |
| } |
| $i = 0; |
| $max = false; |
| foreach ($x as &$y) { |
| if (ipv4_in_cidr($ip.'/'.$cidr_bits, $y)) { |
| $max = $i; |
| } |
| $i++; |
| } |
| return $max; |
| } |
| function ipv4_cidr_split($ipOrCIDR) { |
| $ary = explode('/', $ipOrCIDR, 2); |
| $cidr_bits = isset($ary[1]) ? $ary[1] : IPV4_BITS; |
| if ($cidr_bits > IPV4_BITS) return false; // throw new Exception('CIDR bits > '.IPV4_BITS); |
| $ip = $ary[0]; |
| return array($ip, $cidr_bits); |
| } |
| function ipv4_equals($ipOrCIDRA, $ipOrCIDRB) { |
| return ipv4_normalize_range($ipOrCIDRA) == ipv4_normalize_range($ipOrCIDRB); |
| } |
| function ipv4_cidr_min_ip($ipOrCIDR) { |
| $ary = ipv4_cidr_split($ipOrCIDR); |
| $ipOrCIDR = $ary[0]; |
| $cidr_bits = $ary[1]; |
| if ($cidr_bits > IPV4_BITS) return false; // throw new Exception('CIDR bits > '.IPV4_BITS); |
| $m = ip2bin($ipOrCIDR); |
| $m = substr($m, 0, $cidr_bits) . str_repeat('0', IPV4_BITS-$cidr_bits); |
| return bin2ip($m); |
| } |
| function ipv4_cidr_max_ip($ipOrCIDR) { |
| $ary = ipv4_cidr_split($ipOrCIDR); |
| $ipOrCIDR = $ary[0]; |
| $cidr_bits = $ary[1]; |
| if ($cidr_bits > IPV4_BITS) return false; // throw new Exception('CIDR bits > '.IPV4_BITS); |
| $m = ip2bin($ipOrCIDR); |
| $m = substr($m, 0, $cidr_bits) . str_repeat('1', IPV4_BITS-$cidr_bits); |
| return bin2ip($m); |
| } |
| function ipv4_normalize_range($ipOrCIDR) { |
| $ary = ipv4_cidr_split($ipOrCIDR); |
| $ipOrCIDR = $ary[0]; |
| $cidr_bits = $ary[1]; |
| if ($cidr_bits > IPV4_BITS) return false; // throw new Exception('CIDR bits > '.IPV4_BITS); |
| $m = ip2bin($ipOrCIDR); |
| $m = substr($m, 0, $cidr_bits) . str_repeat('0', IPV4_BITS-$cidr_bits); |
| return bin2ip($m) . '/' . $cidr_bits; |
| } |
| function ipv4_trackdown($ipOrCIDR) { |
| $ary = ipv4_cidr_split($ipOrCIDR); |
| $ipOrCIDR = $ary[0]; |
| $cidr_bits = $ary[1]; |
| if ($cidr_bits > IPV4_BITS) return false; // throw new Exception('CIDR bits > '.IPV4_BITS); |
| $out = array(); |
| $m = ip2bin($ipOrCIDR); |
| for ($i=$cidr_bits; $i>=0; $i--) { |
| $m = substr($m, 0, $i) . str_repeat('0', IPV4_BITS-$i); |
| $out[] = bin2ip($m) . '/' . $i; |
| } |
| return $out; |
| } |
| # --- |
| if (!function_exists('ip2bin')) { |
| function ip2bin($ip) { |
| # Source: http://php.net/manual/en/function.ip2long.php#104163 |
| # modified by VTS |
| if (filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false) { |
| return base_convert(ip2long($ip), 10, 2); |
| } |
| if (filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) === false) { |
| return false; |
| } |
| if (($ip_n = inet_pton($ip)) === false) { |
| return false; |
| } |
| $bits = 15; // 16 x 8 bit = 128bit (ipv6) |
| $ipbin = ''; # added by vts to avoid warning |
| while ($bits >= 0) { |
| $bin = sprintf('%08b', (ord($ip_n[$bits]))); |
| $ipbin = $bin.$ipbin; |
| $bits--; |
| } |
| return $ipbin; |
| } |
| } |
| if (!function_exists('bin2ip')) { |
| function bin2ip($bin) { |
| # Source: http://php.net/manual/en/function.ip2long.php#104163 |
| # modified by VTS |
| if (strlen($bin) <= 32) { // 32bits (ipv4) |
| return long2ip(base_convert($bin, 2, 10)); |
| } |
| if (strlen($bin) != 128) { |
| return false; |
| } |
| $pad = 128 - strlen($bin); |
| for ($i = 1; $i <= $pad; $i++) { |
| $bin = '0'.$bin; |
| } |
| $bits = 0; |
| $ipv6 = ''; # added by vts to avoid warning |
| while ($bits <= 7) { |
| $bin_part = substr($bin,($bits*16),16); |
| $ipv6 .= dechex(bindec($bin_part)) . ':'; |
| $bits++; |
| } |
| return inet_ntop(inet_pton(substr($ipv6, 0, -1))); |
| } |
| } |
| # --- TEST |
| /* |
| assert(ipv4_distance('192.168.0.0/16', '192.168.64.0/18') == -2); |
| assert(ipv4_distance('192.168.0.0/17', '192.168.64.0/18') == -1); |
| assert(ipv4_distance('192.168.64.0/18', '192.168.64.0/18') == 0); |
| assert(ipv4_distance('192.168.64.0/19', '192.168.64.0/18') == 1); |
| assert(ipv4_distance('192.168.64.0/20', '192.168.64.0/18') == 2); |
| assert(ipv4_distance('192.168.69.202/31', '192.168.69.200/31') === false); |
| assert(ipv4_distance('192.168.69.201/32', '192.168.69.200/32') === false); |
| assert(ipv4_distance('192.168.69.201', '192.168.69.200') === false); |
| */ |
| /* |
| $test = '192.168.69.123'; |
| $x = ipv4_trackdown($test); |
| foreach ($x as &$cidr) { |
| $min = ipv4_cidr_min_ip($cidr); |
| $max = ipv4_cidr_max_ip($cidr); |
| echo "$cidr ($min - $max)\n"; |
| } |
| */ |
| /trunk/maintenance/pattern-generator/config.inc.php |
|---|
| 0,0 → 1,16 |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator configuration |
| # |
| # (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2015-04-17 |
| # |
| require_once __DIR__ . '/../../shared/php_includes/config_functions.inc.php'; |
| parse_config(__DIR__ . '/../config/gwhois-pattern-update.conf'); |
| parse_config(__DIR__ . '/../config/urls.conf'); |
| /trunk/maintenance/pattern-generator/generate_asn |
|---|
| 0,0 → 1,119 |
| #!/usr/bin/php |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator: ASN |
| # |
| # (c) 2012 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2012-11-17 |
| # |
| # TODO: also use extended stats of the RIRs, which may have avails / reserved? |
| error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
| require_once __DIR__ . '/config.inc.php'; |
| require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
| require_once __DIR__ . '/rirs.inc.php'; |
| define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
| define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana'); |
| echo "ASN Preparation: Get IANA delegation file\n"; |
| $iana_asn_url = "https://www.iana.org/assignments/as-numbers/as-numbers.txt"; |
| $iana_asn_cache = IANA_CACHE_DIR . '/as-numbers.txt'; |
| if ((USE_CACHE) && (file_exists($iana_asn_cache))) { |
| @mkdir(dirname($iana_asn_cache), 0777, true); |
| $iana_asn_data = file($iana_asn_cache, FILE_IGNORE_NEW_LINES); |
| } else { |
| $iana_asn_data = file($iana_asn_url, FILE_IGNORE_NEW_LINES); |
| } |
| echo "ASN: Parsing...\n"; |
| $version = trim(str_replace('-', '', $iana_asn_data[3])); |
| $iana_asn_data = grep($iana_asn_data, 'whois'); |
| $iana_asn_data = implode("\n", $iana_asn_data); |
| $iana_asn_data = str_replace('Assigned by RIPE NCC', '', $iana_asn_data); |
| $iana_asn_data = str_replace('Assigned by AFRINIC', '', $iana_asn_data); // do not change to 'AfriNIC' ! |
| $iana_asn_data = str_replace('Assigned by LACNIC', '', $iana_asn_data); |
| $iana_asn_data = str_replace('Assigned by APNIC', '', $iana_asn_data); |
| $iana_asn_data = str_replace('Assigned by ARIN', '', $iana_asn_data); |
| while (strpos($iana_asn_data, ' ') !== false) { |
| $iana_asn_data = str_replace(' ', ' ', $iana_asn_data); |
| } |
| $out = ''; |
| $ary = explode("\n", $iana_asn_data); |
| $res = array(); |
| foreach ($ary as &$x) { |
| $x = trim($x); |
| if ($x == '') continue; |
| $bry = explode(" ", $x); |
| $asnums = $bry[0]; |
| $whois = $bry[1]; |
| if (strpos($asnums, '-') === false) { |
| # Single ASN |
| $res[$whois][$asnums] = $asnums; |
| } else { |
| # ASN range |
| $tmp = explode('-', $asnums, 2); |
| $res[$whois][$tmp[0]] = $tmp[1]; |
| } |
| } |
| $out .= "#: version $version\n\n"; |
| $out .= "# Autonomous System Numbers (ASN)\n"; |
| $out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
| $out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
| $out .= "# Source: ($version) $iana_asn_url\n"; |
| $out .= "\n"; |
| foreach ($res as $whois => &$x) { |
| $rir = array_search($whois, $rir_whois_server); |
| if ($asn_additional_params[$rir] != '') { |
| $whois .= '|prefix='.$asn_additional_params[$rir].'|'; |
| } |
| $out .= "# ".strtoupper($rir)."\n"; |
| $out .= ":whois|$whois\n"; |
| foreach ($x as $begin => &$end) { |
| if ($begin == $end) { |
| # Single ASN |
| $out .= "*AS:$begin\n"; |
| } else { |
| # ASN Range |
| $out .= "*AS:$begin-$end\n"; |
| } |
| } |
| $out .= "\n"; |
| } |
| $out .= "# Last resort: ARIN\n"; |
| $out .= ":whois|whois.arin.net|prefix=a |\n"; |
| $out .= "^as(.*)$\n"; |
| echo "ASN: Write to output file\n"; |
| $h = fopen(PATTERN_DIR.'/asn', 'w') or die('Error opening file '.PATTERN_DIR.'/asn'); |
| fwrite($h, $out) or die('Could not write to output file'); |
| fclose($h); |
| echo "ASN Finished\n"; |
| Property changes: |
| Added: svn:executable |
| +* |
| \ No newline at end of property |
| /trunk/maintenance/pattern-generator/generate_domains |
|---|
| 0,0 → 1,161 |
| #!/usr/bin/php |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator: TLDs |
| # |
| # (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2015-07-13 |
| # |
| require_once __DIR__ . '/iana_functions.inc.php'; |
| require_once __DIR__ . '/config.inc.php'; |
| require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
| define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
| define('DOMAINS_PATTERN_FILE', PATTERN_DIR.'/domains'); |
| error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
| $iana_tld_data = null; |
| // Step 1: |
| // Check if in the meantime TLDs without an official whois server were updated to have one |
| // Attention/TODO: A change of the whois name still needs manual intervention! |
| $domains_cont_original = file_get_contents(DOMAINS_PATTERN_FILE); |
| $domains_cont_new = preg_replace_callback( |
| '@# TODO: Entry generated automatically\. Needs manual check\.\n:notice\|\|Whois server unknown \((.*)\)\n\\\.(.*)\$\n@imU', |
| function ($treffer) { |
| $in_all = $treffer[0]; |
| $in_ts = $treffer[1]; |
| $in_tld = $treffer[2]; |
| $days_passed = (time()-strtotime($treffer[1]))/(60*60*24); |
| if ($days_passed < DOMAINS_RECHECK_MISSING_WHOIS_SERVERS) { |
| return $in_all; // leave everything unchanged |
| } |
| $whois_serv = find_rootzone_whois_server($in_tld); |
| if (!$whois_serv) { |
| // Nothing found. Just update last check date. |
| return str_replace($in_ts, date('Y-m-d'), $in_all); |
| } else { |
| // Update the entry |
| return ":whois|$whois_serv\n.${in_tld}\$\n"; |
| } |
| }, |
| $domains_cont_original |
| ); |
| if ($domains_cont_original != $domains_cont_new) { |
| file_put_contents(DOMAINS_PATTERN_FILE, $domains_cont_new); |
| gwi_update_domains_patternfile(); |
| } |
| // Step 2: |
| // Search for new TLDs which are not in our pattern file |
| if (!isset($iana_tld_data)) $iana_tld_data = get_iana_tld_data(); |
| foreach ($iana_tld_data as $tld) { |
| if ($tld[0] == '#') continue; |
| $tld = strtolower($tld); |
| if (!does_exist($tld)) { |
| $whois_serv = find_rootzone_whois_server($tld); |
| $to_append = "\n"; |
| if ($whois_serv) { |
| $to_append .= ":whois|$whois_serv\n"; |
| } else { |
| $to_append .= "# TODO: Entry generated automatically. Needs manual check.\n"; |
| $to_append .= ":notice||Whois server unknown (".date('Y-m-d').")\n"; |
| } |
| $to_append .= "\\.$tld$\n"; |
| file_put_contents(DOMAINS_PATTERN_FILE, $to_append, FILE_APPEND); |
| gwi_update_domains_patternfile(); |
| echo "Added: $tld\n"; |
| } |
| } |
| # ------------------------------------------------------ |
| function does_exist($tld) { |
| $cont = get_united_pattern(); |
| $cont = preg_replace('@#[^\n]*\n@ismU', '', $cont); |
| $tld = strtolower($tld); |
| $cont = strtolower($cont); |
| $cont = str_replace(array('(', ')'), '', $cont); |
| return (strpos($cont, "\\.$tld\$\n") !== false); |
| } |
| function gwi_update_domains_patternfile() { |
| $now = date('Ymd'); |
| $pcont_original = file_get_contents(DOMAINS_PATTERN_FILE); |
| $pcont = $pcont_original; |
| $count = 0; |
| $pcont = preg_replace("@#: version (\\S+)@i", "#: version $now", $pcont, 1, $count); |
| if ($count == 0) { |
| // Add header |
| $pcont = "#: version $now\n". |
| "# Domains\n". |
| "# This file can be updated by ".__DIR__."/generate_domains (only additions of new entries)\n". |
| "# --------------------------------------------------------------------\n". |
| "\n".$pcont; |
| } |
| if ($pcont != $pcont_original) { |
| file_put_contents(DOMAINS_PATTERN_FILE, $pcont); |
| } |
| } |
| function find_rootzone_whois_server($tld) { |
| $whois_serv = iana_get_rootzone_whois_server($tld); |
| // Try to find undocumented whois servers |
| if (TRY_FINDING_HIDDEN_WHOIS_SERVERS) { |
| // TODO: also try out to use the URL of the homepage (in IANAs root DB) |
| if (!$whois_serv) { |
| $check_server = "whois.nic.$tld"; |
| if (gwitc_is_port_open($check_server, 43)) { |
| $whois_serv = $check_server; |
| } |
| } |
| if (!$whois_serv) { |
| $check_server = "whois.$tld"; |
| if (gwitc_is_port_open($check_server, 43)) { |
| $whois_serv = $check_server; |
| } |
| } |
| if (!$whois_serv) { |
| $check_server = "$tld"; |
| if (gwitc_is_port_open($check_server, 43)) { |
| $whois_serv = $check_server; |
| } |
| } |
| } |
| return $whois_serv; |
| } |
| function iana_get_rootzone_whois_server($tld) { |
| $tld = strtolower($tld); |
| $cont = QueryWhoisServer('whois.iana.org', $tld); |
| if (!preg_match('@whois:\\s*(\\S+)@i', $cont, $m)) return false; |
| return $m[1]; |
| } |
| Property changes: |
| Added: svn:executable |
| +* |
| \ No newline at end of property |
| /trunk/maintenance/pattern-generator/generate_ipv4 |
|---|
| 0,0 → 1,190 |
| #!/usr/bin/php |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator: IPv4 |
| # |
| # (c) 2012-2013 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2013-03-08 |
| # |
| error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
| require_once __DIR__ . '/config.inc.php'; |
| require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
| require_once __DIR__ . '/rirs.inc.php'; |
| define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana'); |
| define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
| define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats'); |
| if (USE_IANA_DATA) { |
| echo "IPv4 Preparation: Get IANA delegation file\n"; |
| $iana_root_url = 'ftp://ftp.iana.org/assignments/ipv4-address-space/ipv4-address-space.txt'; |
| $iana_root_cache = IANA_CACHE_DIR . '/ipv4-address-space.txt'; |
| if ((USE_CACHE) && (file_exists($iana_root_cache))) { |
| @mkdir(dirname($iana_root_cache), 0777, true); |
| $iana_root_data = file($iana_root_cache, FILE_IGNORE_NEW_LINES); |
| } else { |
| $iana_root_data = file($iana_root_url, FILE_IGNORE_NEW_LINES); |
| } |
| $iana_root_ver = trim(str_replace('-', '', $iana_root_data[3])); |
| $iana_root_data = grep($iana_root_data, 'whois.'); |
| } |
| foreach ($rirs as &$rir) { |
| $step = 0; |
| $out = ''; |
| $loc_rir_whois_server = $rir_whois_server[$rir]; |
| if (USE_RIR_STAT_DATA) { |
| $step++; |
| echo "IPv4 $rir: Step $step - Get RIR statistics file\n"; |
| $rirstat_url = $rirstat_urls[$rir]; |
| $rirstat_cache = RIRSTATS_CACHE_DIR . "/$rir"; |
| if ((USE_CACHE) && (file_exists($rirstat_cache))) { |
| @mkdir(dirname($rirstat_cache), 0777, true); |
| $rirstat_file = file($rirstat_cache); |
| } else { |
| $rirstat_file = file($rirstat_url); |
| } |
| $rirstat_file = antigrep($rirstat_file, '#'); |
| $tmp = explode('|', $rirstat_file[0]); |
| $rirstat_ver = $tmp[5]; // enddate |
| # $rirstat_ver = substr($rirstat_ver, 0, 4) . '-' . substr($rirstat_ver, 4, 2) . '-' . substr($rirstat_ver, 6, 2); |
| unset($tmp); |
| $out .= "#: version $rirstat_ver\n\n"; |
| } else if (USE_IANA_DATA) { |
| $out .= "#: version $iana_root_ver\n\n"; |
| } else { |
| fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n"); |
| exit(1); |
| } |
| $out .= "# ".strtoupper($rir)." IPv4\n"; |
| $out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
| $out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
| if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) $iana_root_url\n"; |
| if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n"; |
| $out .= "\n"; |
| $out .= ":whois|$loc_rir_whois_server"; |
| if ($ipv4_additional_params[$rir] != '') { |
| $out .= '|prefix='.$ipv4_additional_params[$rir].'|'; |
| } |
| $out .= "\n"; |
| $iana_blocks = array(); |
| if (USE_IANA_DATA) { |
| $step++; |
| echo "IPv4 $rir: Step $step - Parse IANA root blocks\n"; |
| $loc_roots = grep($iana_root_data, $loc_rir_whois_server); |
| $iana_ary = array(); |
| foreach ($loc_roots as &$lr) { |
| $lr = trim($lr); |
| $tmp = explode(' ', $lr, 2); |
| $lr = $tmp[0]; |
| unset($tmp); |
| $lr = preg_replace('|^0*|s', '', $lr); |
| // Break up the CIDR |
| $ary = explode('/', $lr, 2); |
| $bry = ipv4_cidr2range($ary[0], $ary[1]); |
| $iana_blocks[$bry[0]] = $bry[1]; |
| } |
| } |
| $rir_blocks = array(); |
| if (USE_RIR_STAT_DATA) { |
| $step++; |
| echo "IPv4 $rir: Step $step - Parse RIR blocks\n"; |
| # $rirstat_file = grep($rirstat_file, 'ipv4'); |
| # $rirstat_file = antigrep($rirstat_file, 'summary'); |
| ## array_shift($rirstat_file); // header line |
| foreach ($rirstat_file as &$x) { |
| $x = trim($x); |
| if ($x == '') continue; |
| $ary = explode('|', $x); |
| # arin|* |asn|* |22244|summary |
| # ^0 ^1 ^2 ^3 ^4 ^5 |
| if ($ary[5] == 'summary') continue; |
| # Example: |
| # afrinic|ZM|ipv4|196.46.192.0|8192|20031024|allocated |
| # ^0 ^1 ^2 ^3 ^4 ^5 ^6 |
| # 2.1AP resp 2.3 (= extended) are compatible with 2.0 |
| $type = $ary[2]; |
| if ($type != 'ipv4') continue; |
| $baseip = $ary[3]; |
| $nums = $ary[4]; |
| # $state = $ary[6]; |
| $topip = ipv4_add($baseip, $nums-1); |
| $rir_blocks[$baseip] = $topip; |
| } |
| } |
| if (USE_IANA_DATA && USE_RIR_STAT_DATA) { |
| # Use both |
| $step++; |
| echo "IPv4 $rir: Step $step - Merging RIR and IANA stats\n"; |
| $blocks_merged = ipv4_merge_arrays($rir_blocks, $iana_blocks); |
| } else if (USE_IANA_DATA) { |
| # Use only IANA |
| $blocks_merged = $iana_blocks; |
| } else if (USE_RIR_STAT_DATA) { |
| # Use only RIR data |
| $blocks_merged = $rir_blocks; |
| } else { |
| fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n"); |
| exit(1); |
| } |
| $step++; |
| $blocks_merged = ipv4_merge_address_blocks($blocks_merged, "IPv4 $rir: Step $step - "); |
| // --- CIDR finden und ausgeben --- |
| $step++; |
| echo "IPv4 $rir: Step $step - Analyze address ranges and write CIDR blocks\n"; |
| foreach ($blocks_merged as $baseip => &$topip) { |
| if (INCLUDE_BLOCK_RANGE_COMMENTS) $out .= "# $baseip - $topip\n"; |
| $cidrs = ipv4_range2cidr($baseip, $topip, IPV4_SHORT_FORM); |
| foreach ($cidrs as &$cidr) { |
| $out .= "=$cidr\n"; |
| } |
| } |
| # --- Write to file --- |
| $step++; |
| echo "IPv4 $rir: Step $step - Write to output file\n"; |
| $h = fopen(PATTERN_DIR."/ipv4_$rir", 'w') or die("Error opening file ".PATTERN_DIR."/ipv4_$rir"); |
| fwrite($h, $out) or die('Could not write to output file'); |
| fclose($h); |
| } |
| echo "IPv4 Finished!\n"; |
| Property changes: |
| Added: svn:executable |
| +* |
| \ No newline at end of property |
| /trunk/maintenance/pattern-generator/generate_ipv6 |
|---|
| 0,0 → 1,191 |
| #!/usr/bin/php |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator: IPv6 |
| # |
| # (c) 2012-2013 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2013-03-08 |
| # |
| error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
| require_once __DIR__ . '/config.inc.php'; |
| require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
| require_once __DIR__ . '/rirs.inc.php'; |
| define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana'); |
| define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
| define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats'); |
| if (USE_IANA_DATA) { |
| echo "IPv6 Preparation: Get IANA delegation file\n"; |
| $iana_root_url = 'ftp://ftp.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.txt'; |
| $iana_root_cache = IANA_CACHE_DIR . '/ipv6-unicast-address-assignments.txt'; |
| if ((USE_CACHE) && (file_exists($iana_root_cache))) { |
| @mkdir(dirname($iana_root_cache), 0777, true); |
| $iana_root_data = file($iana_root_cache, FILE_IGNORE_NEW_LINES); |
| } else { |
| $iana_root_data = file($iana_root_url, FILE_IGNORE_NEW_LINES); |
| } |
| $iana_root_ver = trim(str_replace('-', '', $iana_root_data[3])); |
| $iana_root_data = grep($iana_root_data, 'whois.'); |
| } |
| foreach ($rirs as &$rir) { |
| $step = 0; |
| $out = ''; |
| $loc_rir_whois_server = $rir_whois_server[$rir]; |
| if (USE_RIR_STAT_DATA) { |
| $step++; |
| echo "IPv6 $rir: Step $step - Get RIR statistics file\n"; |
| $rirstat_url = $rirstat_urls[$rir]; |
| $rirstat_cache = RIRSTATS_CACHE_DIR . "/$rir"; |
| if ((USE_CACHE) && (file_exists($rirstat_cache))) { |
| @mkdir(dirname($rirstat_cache), 0777, true); |
| $rirstat_file = file($rirstat_cache); |
| } else { |
| $rirstat_file = file($rirstat_url); |
| } |
| $rirstat_file = antigrep($rirstat_file, '#'); |
| $tmp = explode('|', $rirstat_file[0]); |
| $rirstat_ver = $tmp[5]; // enddate |
| # $rirstat_ver = substr($rirstat_ver, 0, 4) . '-' . substr($rirstat_ver, 4, 2) . '-' . substr($rirstat_ver, 6, 2); |
| unset($tmp); |
| $out .= "#: version $rirstat_ver\n\n"; |
| } else if (USE_IANA_DATA) { |
| $out .= "#: version $iana_root_ver\n\n"; |
| } else { |
| fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n"); |
| exit(1); |
| } |
| $out .= "# ".strtoupper($rir)." IPv6\n"; |
| $out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
| $out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
| if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) $iana_root_url\n"; |
| if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n"; |
| $out .= "\n"; |
| $out .= ":whois|$loc_rir_whois_server"; |
| if ($ipv6_additional_params[$rir] != '') { |
| $out .= '|prefix='.$ipv6_additional_params[$rir].'|'; |
| } |
| $out .= "\n"; |
| $iana_blocks = array(); |
| if (USE_IANA_DATA) { |
| $step++; |
| echo "IPv6 $rir: Step $step - Parse IANA root blocks\n"; |
| $loc_roots = grep($iana_root_data, $loc_rir_whois_server); |
| $iana_ary = array(); |
| foreach ($loc_roots as &$lr) { |
| $lr = trim($lr); |
| $tmp = explode(' ', $lr, 2); |
| $lr = $tmp[0]; |
| unset($tmp); |
| $lr = preg_replace('|^0*|s', '', $lr); |
| // Break up the CIDR |
| $ary = explode('/', $lr, 2); |
| $bry = ipv6_cidr2range($ary[0], $ary[1]); |
| $iana_blocks[$bry[0]] = $bry[1]; |
| } |
| } |
| $rir_blocks = array(); |
| if (USE_RIR_STAT_DATA) { |
| $step++; |
| echo "IPv6 $rir: Step $step - Parse RIR blocks\n"; |
| # $rirstat_file = grep($rirstat_file, 'ipv6'); |
| # $rirstat_file = antigrep($rirstat_file, 'summary'); |
| ## array_shift($rirstat_file); // header line |
| foreach ($rirstat_file as &$x) { |
| $x = trim($x); |
| if ($x == '') continue; |
| $ary = explode('|', $x); |
| # arin|* |asn|* |22244|summary |
| # ^0 ^1 ^2 ^3 ^4 ^5 |
| if ($ary[5] == 'summary') continue; |
| # Example: |
| # afrinic|KE|ipv6|2c0f:fe88::|32|20100728|allocated |
| # ^0 ^1 ^2 ^3 ^4 ^5 ^6 |
| # 2.1AP resp 2.3 (= extended) are compatible with 2.0 |
| $type = $ary[2]; |
| if ($type != 'ipv6') continue; |
| $baseip = $ary[3]; |
| $cidr = $ary[4]; |
| # $state = $ary[6]; |
| $range = ipv6_cidr2range($baseip, $cidr); |
| $topip = $range[1]; |
| $rir_blocks[$baseip] = $topip; |
| } |
| } |
| if (USE_IANA_DATA && USE_RIR_STAT_DATA) { |
| # Use both |
| $step++; |
| echo "IPv6 $rir: Step $step - Merging RIR and IANA stats\n"; |
| $blocks_merged = ipv6_merge_arrays($rir_blocks, $iana_blocks); |
| } else if (USE_IANA_DATA) { |
| # Use only IANA |
| $blocks_merged = $iana_blocks; |
| } else if (USE_RIR_STAT_DATA) { |
| # Use only RIR data |
| $blocks_merged = $rir_blocks; |
| } else { |
| fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n"); |
| exit(1); |
| } |
| $step++; |
| $blocks_merged = ipv6_merge_address_blocks($blocks_merged, "IPv6 $rir: Step $step - "); |
| // --- CIDR finden und ausgeben --- |
| $step++; |
| echo "IPv6 $rir: Step $step - Analyze address ranges and write CIDR blocks\n"; |
| foreach ($blocks_merged as $baseip => &$topip) { |
| if (INCLUDE_BLOCK_RANGE_COMMENTS) $out .= "# $baseip - $topip\n"; |
| $cidrs = ipv6_range2cidr($baseip, $topip); |
| foreach ($cidrs as &$cidr) { |
| $out .= "=$cidr\n"; |
| } |
| } |
| # --- Write to file --- |
| $step++; |
| echo "IPv6 $rir: Step $step - Write to output file\n"; |
| $h = fopen(PATTERN_DIR."/ipv6_$rir", 'w') or die("Error opening file ".PATTERN_DIR."/ipv6_$rir"); |
| fwrite($h, $out) or die('Could not write to output file'); |
| fclose($h); |
| } |
| echo "IPv6 Finished!\n"; |
| Property changes: |
| Added: svn:executable |
| +* |
| \ No newline at end of property |
| /trunk/maintenance/pattern-generator/generate_tld |
|---|
| 0,0 → 1,116 |
| #!/usr/bin/php |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator: TLD |
| # |
| # (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2015-05-07 |
| # |
| error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
| require_once __DIR__ . '/config.inc.php'; |
| require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
| require_once __DIR__ . '/rirs.inc.php'; |
| require_once __DIR__ . '/iana_functions.inc.php'; |
| define('PATTERN_DIR', __DIR__ . '/../../main/pattern'); |
| define('CACHE_FILE_DIR', __DIR__ . '/../.cache/cache'); |
| $out = ''; |
| $out .= "# TLDs\n"; |
| $out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
| $out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
| $out .= "\n"; |
| $max_version = 0; |
| # --- |
| echo "Deleted TLD: Get deleted TLD list (".DELETED_TLD_LIST.")\n"; |
| $iana_tld_data = cached_file(DELETED_TLD_LIST, CACHE_FILE_DIR); |
| $iana_tld_data = str_replace("\r", '', $iana_tld_data); |
| $iana_tld_data = trim($iana_tld_data); |
| $iana_tld_data = explode("\n", $iana_tld_data); |
| echo "Deleted TLD: Parsing...\n"; |
| if (preg_match('@# Latest update detected on (.*)@', $iana_tld_data[0], $m)) { |
| $version = $m[1]; |
| $version = substr($version, 0, 10); |
| $version = str_replace('-', '', $version); |
| if ($version > $max_version) $max_version = $version; |
| } |
| array_shift($iana_tld_data); |
| $tlds = implode('|', $iana_tld_data); |
| $iana_tld_data = implode("\n", $iana_tld_data); |
| $out .= "# Source: ($version) ".DELETED_TLD_LIST."\n"; |
| $out .= "\n"; |
| $out .= ":notice||Disallocated by IANA"; |
| if ($tld_additional_params['iana'] != '') { |
| $out .= '|prefix='.$tld_additional_params['iana'].'|'; |
| } |
| $out .= "\n"; |
| $out .= "^(tld\\s){0,1}($tlds)\$\n"; |
| $out .= "\n"; |
| # --- |
| echo "TLD: Get IANA delegation file (".IANA_TLD_REGISTRY.")\n"; |
| $iana_tld_data = get_iana_tld_data(); |
| echo "TLD: Parsing...\n"; |
| if (preg_match('@# Version (\d+), Last Updated (.*)@', $iana_tld_data[0], $m)) { |
| $version = $m[1]; |
| $version = substr($version, 0, 8); |
| if ($version > $max_version) $max_version = $version; |
| } |
| array_shift($iana_tld_data); |
| $tlds = implode('|', $iana_tld_data); |
| $iana_tld_data = implode("\n", $iana_tld_data); |
| $out .= "# Source: ($version) ".IANA_TLD_REGISTRY."\n"; |
| $out .= "\n"; |
| $out .= ":whois|".$rir_whois_server['iana']; |
| if ($tld_additional_params['iana'] != '') { |
| $out .= '|prefix='.$tld_additional_params['iana'].'|'; |
| } |
| $out .= "\n"; |
| $out .= "^($tlds)\$\n"; |
| $out .= "\n"; |
| $out .= ":whois|".$rir_whois_server['iana']."|strip=tld "; |
| if ($tld_additional_params['iana'] != '') { |
| $out .= '|prefix='.$tld_additional_params['iana']; |
| } |
| $out .= "|\n"; |
| $out .= "^tld\\s\\S+\$\n"; |
| $out .= "\n"; |
| # --- |
| $out = "#: version $max_version\n\n".$out; |
| echo "TLD-Generator: Write to output file\n"; |
| $h = fopen(PATTERN_DIR.'/tld', 'w') or die('Error opening file '.PATTERN_DIR.'/tld'); |
| fwrite($h, $out) or die('Could not write to output file'); |
| fclose($h); |
| echo "TLD-Generator: Finished\n"; |
| Property changes: |
| Added: svn:executable |
| +* |
| \ No newline at end of property |
| /trunk/maintenance/pattern-generator/gwhois-pattern-update |
|---|
| 0,0 → 1,104 |
| #!/bin/bash |
| # |
| # generic Whois - Automatic Pattern Generator Master-Script |
| # |
| # (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2015-04-15 |
| # |
| # TODO: "force" option will enforce generate even when web download was not http 200 |
| DIR=$( dirname "$0" ); |
| . "$DIR"/../config/gwhois-pattern-update.conf |
| . "$DIR"/../config/urls.conf |
| IANA_CACHE_DIR="$DIR"/../.cache/pattern-generator/iana |
| function download() { |
| # in: <url> <outputfile> |
| # out: 0 = success |
| # 1 = error |
| # 2 = success, but nothing updated |
| # output file will not be touched when nothing needs to be updated |
| # Thanks to http://blog.yjl.im/2012/03/downloading-only-when-modified-using.html for this wonderful tutorial |
| if [ -f "$2" ]; then |
| HTTPCODE=$( curl -o "$2" -z "$2" -fsSL --write-out %{http_code} -- "$1" ) |
| else |
| HTTPCODE=$( curl -o "$2" -fsSL --write-out %{http_code} -- "$1" ) |
| fi |
| RES=$? |
| if [ $RES -eq 0 ]; then |
| touch "$2.success" |
| if [[ "$HTTPCODE" == "200" ]]; then |
| return 0 |
| else |
| return 2 |
| fi |
| else |
| touch "$2.fail" |
| return 1 |
| fi |
| } |
| if [ $? -ne 0 ] |
| then |
| echo "Error when reading configuration value IANA_CACHE_DIR" > /dev/stderr |
| exit 1 |
| fi |
| echo "=================== GWHOIS PATTERN UPDATE ===================" |
| echo "This script downloads the latest RIR statistics" |
| echo "and renews the pattern files of GWhoIs to provide" |
| echo "up-to-date informations of IPv4, IPv6 and ASN resources." |
| echo "ATTENTION: The parsing of the downloaded files will take" |
| echo "approximately 15-20 minutes!!!" |
| echo "=============================================================" |
| # RIR stats are necessary for generate_ipv4 and generate_ipv6 |
| echo "RIR stats Synchronization ..."; |
| "$DIR"/sync_rir_stats |
| mkdir -p "$IANA_CACHE_DIR"/ |
| echo "IPv4 Synchronization ..."; |
| download "$IANA_IPV4_ASSIGNMENTS" "$IANA_CACHE_DIR"/ipv4-address-space.txt |
| if [ $? -eq 0 ]; then |
| echo "IPv4 Generating ..." |
| "$DIR"/generate_ipv4 |
| fi |
| echo "IPv6 Synchronization ..."; |
| download "$IANA_IPV6_ASSIGNMENTS" "$IANA_CACHE_DIR"/ipv6-unicast-address-assignments.txt |
| if [ $? -eq 0 ]; then |
| echo "IPv6 Generating ..." |
| "$DIR"/generate_ipv6 |
| fi |
| echo "ASN Synchronization ..."; |
| download "$IANA_AS_NUMBERS" "$IANA_CACHE_DIR"/as-numbers.txt |
| if [ $? -eq 0 ]; then |
| echo "ASN Generating ..." |
| "$DIR"/generate_asn |
| fi |
| echo "TLD Synchronization ..."; |
| download "$IANA_TLD_REGISTRY" "$IANA_CACHE_DIR"/tlds-alpha-by-domain.txt |
| if [ $? -eq 0 ]; then |
| echo "TLD Generating ..." |
| "$DIR"/generate_tld |
| echo "Domains Generating ..." |
| "$DIR"/generate_domains |
| fi |
| echo "Everything finished!" |
| Property changes: |
| Added: svn:executable |
| +* |
| \ No newline at end of property |
| /trunk/maintenance/pattern-generator/iana_functions.inc.php |
|---|
| 0,0 → 1,26 |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator: TLD |
| # |
| # (c) 2012 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2014-04-31 |
| # |
| require_once __DIR__ . '/config.inc.php'; |
| define('IANA_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/iana'); |
| define('IANA_TLD_CACHE', IANA_CACHE_DIR . '/tlds-alpha-by-domain.txt'); |
| function get_iana_tld_data() { |
| if ((USE_CACHE) && (file_exists(IANA_TLD_CACHE))) { |
| @mkdir(dirname(IANA_TLD_CACHE), 0777, true); |
| return file(IANA_TLD_CACHE, FILE_IGNORE_NEW_LINES); |
| } else { |
| return file(IANA_TLD_REGISTRY, FILE_IGNORE_NEW_LINES); |
| } |
| } |
| /trunk/maintenance/pattern-generator/info |
|---|
| 0,0 → 1,8 |
| This tool updates the GWhoIs pattern files which include the following patterns: |
| - IPv4 blocks |
| - IPv6 blocks |
| - ASNs |
| - TLDs |
| - new gTLDs |
| You can run it as cronjob |
| /trunk/maintenance/pattern-generator/rir_bugs.info |
|---|
| Cannot display: file marked as a binary type. |
| svn:mime-type = application/x-info |
| Property changes: |
| Added: svn:mime-type |
| +application/x-info |
| \ No newline at end of property |
| /trunk/maintenance/pattern-generator/rirs.inc.php |
|---|
| 0,0 → 1,104 |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator RIR file |
| # |
| # (c) 2012-2013 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2013-03-07 |
| # |
| $rirs = array('afrinic', 'apnic', 'arin', 'lacnic', 'ripencc'); |
| # IANA is not a "real" RIR - to enable, please add "iana" manually: |
| # $rirs[] = 'iana'; |
| // Extended format also has "reserved" and "available" resources! |
| // I have contacted the NRO EC in Apr 2012 to suggest that the other RIRs also use that extended format, |
| // so that the statistics can also show available and reserved blocks |
| // APNIC supports extended RIR statistics exchange format since 2008-02-14 . |
| // RIPE NCC supports extended RIR statistics exchange format since 2012-05-23 . |
| // LACNIC supports extended RIR statistics exchange format since 2012-06-28 . |
| // AfriNIC supports extended RIR statistics exchange format since 2012-10-02 . |
| // ARIN supports extended RIR statistics exchange format since 2013-03-05 . (First publication 2013-02-16 , revoked due to problems ) |
| $supports_extended_rirstat = array(); |
| $supports_extended_rirstat['afrinic'] = true; |
| $supports_extended_rirstat['apnic'] = true; |
| $supports_extended_rirstat['arin'] = true; // must always be true, since they discontinued the old stats |
| $supports_extended_rirstat['lacnic'] = true; |
| $supports_extended_rirstat['ripencc'] = true; |
| $supports_extended_rirstat['iana'] = false; |
| $rir_whois_server = array(); |
| $rir_whois_server['afrinic'] = 'whois.afrinic.net'; |
| $rir_whois_server['apnic'] = 'whois.apnic.net'; |
| $rir_whois_server['arin'] = 'whois.arin.net'; |
| $rir_whois_server['lacnic'] = 'whois.lacnic.net'; |
| $rir_whois_server['ripencc'] = 'whois.ripe.net'; |
| $rir_whois_server['iana'] = 'whois.iana.org'; |
| $rir_domain = array(); |
| $rir_domain['afrinic'] = 'afrinic.net'; |
| $rir_domain['apnic'] = 'apnic.net'; |
| $rir_domain['arin'] = 'arin.net'; |
| $rir_domain['lacnic'] = 'lacnic.net'; |
| $rir_domain['ripencc'] = 'ripe.net'; |
| $rir_domain['iana'] = 'iana.org'; |
| // If the RIR offers and extended format, use it instead! |
| // Alternative address: ftp://ftp.iana.org/pub/mirror/rirstats/ |
| $rirstat_urls = array(); |
| $rirstat_urls['afrinic'] = 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest'; |
| $rirstat_urls['apnic'] = 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-extended-latest'; |
| $rirstat_urls['arin'] = 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest'; |
| $rirstat_urls['lacnic'] = 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest'; |
| $rirstat_urls['ripencc'] = 'ftp://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest'; |
| $rirstat_urls['iana'] = 'ftp://ftp.apnic.net/pub/stats/iana/delegated-iana-latest'; # This is a service by apnic.net . IANA is not running a public service, NRO EC resolution needed |
| $ipv4_additional_params = array(); |
| $ipv4_additional_params['afrinic'] = ''; |
| $ipv4_additional_params['apnic'] = ''; |
| $ipv4_additional_params['arin'] = 'n '; |
| $ipv4_additional_params['lacnic'] = ''; |
| $ipv4_additional_params['ripencc'] = ''; |
| $ipv4_additional_params['iana'] = ''; |
| $ipv6_additional_params = array(); |
| $ipv6_additional_params['afrinic'] = ''; |
| $ipv6_additional_params['apnic'] = ''; |
| $ipv6_additional_params['arin'] = 'n '; |
| $ipv6_additional_params['lacnic'] = ''; |
| $ipv6_additional_params['ripencc'] = ''; |
| $ipv6_additional_params['iana'] = ''; |
| $asn_additional_params = array(); |
| $asn_additional_params['afrinic'] = ''; |
| $asn_additional_params['apnic'] = ''; |
| $asn_additional_params['arin'] = 'a '; |
| $asn_additional_params['lacnic'] = ''; |
| $asn_additional_params['ripencc'] = ''; |
| $asn_additional_params['iana'] = ''; |
| $tld_additional_params = array(); |
| $tld_additional_params['afrinic'] = ''; |
| $tld_additional_params['apnic'] = ''; |
| $tld_additional_params['arin'] = 'z '; # ??? |
| $tld_additional_params['lacnic'] = ''; |
| $tld_additional_params['ripencc'] = ''; |
| $tld_additional_params['iana'] = ''; |
| // Output unfiltered results for ARIN |
| if ((defined('ENABLE_UNFILTERED_ARIN_OUTPUT')) && (ENABLE_UNFILTERED_ARIN_OUTPUT)) { |
| $ipv6_additional_params['arin'] .= '+ '; |
| $ipv4_additional_params['arin'] .= '+ '; |
| $asn_additional_params['arin'] .= '+ '; |
| } |
| // Output unfiltered results for RIPE |
| if ((defined('ENABLE_UNFILTERED_RIPE_OUTPUT')) && (ENABLE_UNFILTERED_RIPE_OUTPUT)) { |
| $ipv6_additional_params['ripencc'] .= '-B '; |
| $ipv4_additional_params['ripencc'] .= '-B '; |
| $asn_additional_params['ripencc'] .= '-B '; |
| } |
| /trunk/maintenance/pattern-generator/sync_rir_stats |
|---|
| 0,0 → 1,97 |
| #!/usr/bin/php |
| <?php |
| # |
| # generic Whois - Automatic Pattern Generator RIR-Statistics-Synchronisation-Tool |
| # |
| # (c) 2012-2015 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de] |
| # |
| # Distribution, usage etc. pp. regulated by the current version of GPL. |
| # |
| # |
| # Version 2015-04-14 |
| # |
| error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
| require_once __DIR__ . '/config.inc.php'; |
| require_once __DIR__ . '/rirs.inc.php'; |
| define('RIRSTATS_CACHE_DIR', __DIR__ . '/../.cache/pattern-generator/rirstats'); |
| function rir_get_md5_sum($url) { |
| // MD5 (/var/opt/ftp/pub/apnic/stats/apnic/delegated-apnic-extended-latest) = 82c291bb5d4363a3db254853c1602777 |
| // MD5 (delegated-lacnic-extended-latest) = 2815f0b5837d5658acf1659dff98bb52 |
| // MD5 (delegated-ripencc-extended-latest) = d4caddde59952c44b584cacf720ef836 |
| // MD5 (delegated-arin-latest) = 3137d635d5e647481af972ddc87e5570 |
| // MD5 (delegated-afrinic-latest) = 9c1cd55e8894402062282e3ebdcf53c8 <-- WRONG FILE NAME SHOWN FOR delegated-afrinic-extended-latest.md5! |
| $md5_cont = file_get_contents($url.'.md5'); |
| if (substr($md5_cont, 0, 5) != 'MD5 (') { |
| // Some older *.md5 files of AfriNIC have following formats: |
| // 73d5e32afd43eac0beb4635b6a9056c4 delegated-afrinic-latest |
| // since 2012-06-25 AfriNIC uses the "normal" format |
| // MD5 (delegated-afrinic-latest) = 9c1cd55e8894402062282e3ebdcf53c8 |
| // however, the filename is wrong (delegated-afrinic-latest instead of delegated-afrinic-extended-latest) |
| return substr($md5_cont, 0, 32); |
| } else { |
| // Die anderen RIRs |
| $tmp = explode(' = ', $md5_cont); |
| return trim($tmp[1]); |
| } |
| } |
| function rir_download_report($url, $outfile, $do_md5_check = true) { |
| if (($do_md5_check) && (file_exists($outfile))) { |
| $md5_ist = md5_file($outfile); |
| $md5_soll = rir_get_md5_sum($url); |
| if ($md5_soll == $md5_ist) { |
| touch("$outfile.success"); |
| return true; |
| } |
| } |
| $cont = file_get_contents($url); |
| if (!$cont) { |
| touch("$outfile.fail"); |
| return false; |
| } |
| if ($do_md5_check) { |
| $md5_ist = md5($cont); |
| if (!isset($md5_soll)) $md5_soll = rir_get_md5_sum($url); |
| if ($md5_soll != $md5_ist) { |
| touch("$outfile.fail"); |
| return false; |
| } |
| } |
| $h = fopen($outfile, 'w'); |
| if (!$h) return false; |
| if (!fwrite($h, $cont)) { |
| touch("$outfile.fail"); |
| return false; |
| } |
| fclose($h); |
| touch("$outfile.success"); |
| return true; |
| } |
| $rirs[] = 'iana'; |
| @mkdir(RIRSTATS_CACHE_DIR.'/', 0777, true); |
| foreach ($rirs as $rir) { |
| $url = $rirstat_urls[$rir]; |
| $failcounter = 0; |
| while (!rir_download_report($url, RIRSTATS_CACHE_DIR."/$rir", $rir != 'iana')) { |
| $failcounter++; |
| echo "Retry downloading $rir stats ($failcounter)...\n"; |
| if ($failcounter > 100) break; |
| sleep(60); |
| } |
| } |
| Property changes: |
| Added: svn:executable |
| +* |
| \ No newline at end of property |