Rev 5 | Rev 77 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 5 | Rev 11 | ||
---|---|---|---|
1 | #!/usr/bin/php |
1 | #!/usr/bin/php |
2 | <?php |
2 | <?php |
3 | 3 | ||
4 | # |
4 | # |
5 | # VWhois (ViaThinkSoft WHOIS, a fork of generic Whois / gwhois) |
5 | # VGWhoIs (ViaThinkSoft Global WhoIs, a fork of generic Whois / gwhois) |
6 | # Maintenance / Developer utilities |
6 | # Maintenance / Developer utilities |
7 | # |
7 | # |
8 | # (c) 2012-2019 by Daniel Marschall, ViaThinkSoft <info@daniel-marschall.de> |
8 | # (c) 2012-2019 by Daniel Marschall, ViaThinkSoft <info@daniel-marschall.de> |
9 | # |
9 | # |
10 | # License: https://www.gnu.org/licenses/gpl-2.0.html (GPL version 2) |
10 | # License: https://www.gnu.org/licenses/gpl-2.0.html (GPL version 2) |
11 | # |
11 | # |
12 | 12 | ||
13 | error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
13 | error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED); |
14 | 14 | ||
15 | require_once __DIR__ . '/config.inc.php'; |
15 | require_once __DIR__ . '/config.inc.php'; |
16 | require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
16 | require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php'; |
17 | require_once __DIR__ . '/rirs.inc.php'; |
17 | require_once __DIR__ . '/rirs.inc.php'; |
18 | 18 | ||
19 | if (USE_IANA_DATA) { |
19 | if (USE_IANA_DATA) { |
20 | echo "IPv4 Preparation: Get IANA delegation file\n"; |
20 | echo "IPv4 Preparation: Get IANA delegation file\n"; |
21 | 21 | ||
22 | $iana_root_data = explode("\n", cached_file(IANA_IPV4_ASSIGNMENTS, CACHE_FILE_DIR)); |
22 | $iana_root_data = explode("\n", cached_file(IANA_IPV4_ASSIGNMENTS, CACHE_FILE_DIR)); |
23 | $iana_root_data = array_map('trim', $iana_root_data); |
23 | $iana_root_data = array_map('trim', $iana_root_data); |
24 | 24 | ||
25 | $iana_root_ver = trim(str_replace('-', '', $iana_root_data[3])); |
25 | $iana_root_ver = trim(str_replace('-', '', $iana_root_data[3])); |
26 | $iana_root_data = grep($iana_root_data, 'whois.'); |
26 | $iana_root_data = grep($iana_root_data, 'whois.'); |
27 | } |
27 | } |
28 | 28 | ||
29 | foreach ($rirs as &$rir) { |
29 | foreach ($rirs as &$rir) { |
30 | $step = 0; |
30 | $step = 0; |
31 | $out = ''; |
31 | $out = ''; |
32 | 32 | ||
33 | $loc_rir_whois_server = $rir_whois_server[$rir]; |
33 | $loc_rir_whois_server = $rir_whois_server[$rir]; |
34 | 34 | ||
35 | if (USE_RIR_STAT_DATA) { |
35 | if (USE_RIR_STAT_DATA) { |
36 | $step++; |
36 | $step++; |
37 | echo "IPv4 $rir: Step $step - Get RIR statistics file\n"; |
37 | echo "IPv4 $rir: Step $step - Get RIR statistics file\n"; |
38 | 38 | ||
39 | $rirstat_url = $rirstat_urls[$rir]; |
39 | $rirstat_url = $rirstat_urls[$rir]; |
40 | $rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR)); |
40 | $rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR)); |
41 | 41 | ||
42 | $rirstat_file = antigrep($rirstat_file, '#'); |
42 | $rirstat_file = antigrep($rirstat_file, '#'); |
43 | 43 | ||
44 | $tmp = explode('|', $rirstat_file[0]); |
44 | $tmp = explode('|', $rirstat_file[0]); |
45 | $rirstat_ver = $tmp[5]; // enddate |
45 | $rirstat_ver = $tmp[5]; // enddate |
46 | # $rirstat_ver = substr($rirstat_ver, 0, 4) . '-' . substr($rirstat_ver, 4, 2) . '-' . substr($rirstat_ver, 6, 2); |
46 | # $rirstat_ver = substr($rirstat_ver, 0, 4) . '-' . substr($rirstat_ver, 4, 2) . '-' . substr($rirstat_ver, 6, 2); |
47 | unset($tmp); |
47 | unset($tmp); |
48 | 48 | ||
49 | $out .= "#: version $rirstat_ver\n\n"; |
49 | $out .= "#: version $rirstat_ver\n\n"; |
50 | } else if (USE_IANA_DATA) { |
50 | } else if (USE_IANA_DATA) { |
51 | $out .= "#: version $iana_root_ver\n\n"; |
51 | $out .= "#: version $iana_root_ver\n\n"; |
52 | } else { |
52 | } else { |
53 | fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n"); |
53 | fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n"); |
54 | exit(1); |
54 | exit(1); |
55 | } |
55 | } |
56 | 56 | ||
57 | $out .= "# ".strtoupper($rir)." IPv4\n"; |
57 | $out .= "# ".strtoupper($rir)." IPv4\n"; |
58 | $out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n"; |
58 | $out .= "# Automatically generated by ".__DIR__."/vgwhois-pattern-update\n"; |
59 | $out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
59 | $out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n"; |
60 | if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) ".IANA_IPV4_ASSIGNMENTS."\n"; |
60 | if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) ".IANA_IPV4_ASSIGNMENTS."\n"; |
61 | if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n"; |
61 | if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n"; |
62 | $out .= "\n"; |
62 | $out .= "\n"; |
63 | 63 | ||
64 | $out .= ":whois|$loc_rir_whois_server"; |
64 | $out .= ":whois|$loc_rir_whois_server"; |
65 | if ($ipv4_additional_params[$rir] != '') { |
65 | if ($ipv4_additional_params[$rir] != '') { |
66 | $out .= '|prefix='.$ipv4_additional_params[$rir].'|'; |
66 | $out .= '|prefix='.$ipv4_additional_params[$rir].'|'; |
67 | } |
67 | } |
68 | $out .= "\n"; |
68 | $out .= "\n"; |
69 | 69 | ||
70 | $iana_blocks = array(); |
70 | $iana_blocks = array(); |
71 | if (USE_IANA_DATA) { |
71 | if (USE_IANA_DATA) { |
72 | $step++; |
72 | $step++; |
73 | echo "IPv4 $rir: Step $step - Parse IANA root blocks\n"; |
73 | echo "IPv4 $rir: Step $step - Parse IANA root blocks\n"; |
74 | 74 | ||
75 | $loc_roots = grep($iana_root_data, $loc_rir_whois_server); |
75 | $loc_roots = grep($iana_root_data, $loc_rir_whois_server); |
76 | 76 | ||
77 | $iana_ary = array(); |
77 | $iana_ary = array(); |
78 | foreach ($loc_roots as &$lr) { |
78 | foreach ($loc_roots as &$lr) { |
79 | $lr = trim($lr); |
79 | $lr = trim($lr); |
80 | $tmp = explode(' ', $lr, 2); |
80 | $tmp = explode(' ', $lr, 2); |
81 | $lr = $tmp[0]; |
81 | $lr = $tmp[0]; |
82 | unset($tmp); |
82 | unset($tmp); |
83 | $lr = preg_replace('|^0*|s', '', $lr); |
83 | $lr = preg_replace('|^0*|s', '', $lr); |
84 | 84 | ||
85 | // Break up the CIDR |
85 | // Break up the CIDR |
86 | $ary = explode('/', $lr, 2); |
86 | $ary = explode('/', $lr, 2); |
87 | $bry = ipv4_cidr2range($ary[0], $ary[1]); |
87 | $bry = ipv4_cidr2range($ary[0], $ary[1]); |
88 | $iana_blocks[$bry[0]] = $bry[1]; |
88 | $iana_blocks[$bry[0]] = $bry[1]; |
89 | } |
89 | } |
90 | } |
90 | } |
91 | 91 | ||
92 | $rir_blocks = array(); |
92 | $rir_blocks = array(); |
93 | if (USE_RIR_STAT_DATA) { |
93 | if (USE_RIR_STAT_DATA) { |
94 | $step++; |
94 | $step++; |
95 | echo "IPv4 $rir: Step $step - Parse RIR blocks\n"; |
95 | echo "IPv4 $rir: Step $step - Parse RIR blocks\n"; |
96 | 96 | ||
97 | # $rirstat_file = grep($rirstat_file, 'ipv4'); |
97 | # $rirstat_file = grep($rirstat_file, 'ipv4'); |
98 | # $rirstat_file = antigrep($rirstat_file, 'summary'); |
98 | # $rirstat_file = antigrep($rirstat_file, 'summary'); |
99 | ## array_shift($rirstat_file); // header line |
99 | ## array_shift($rirstat_file); // header line |
100 | 100 | ||
101 | foreach ($rirstat_file as &$x) { |
101 | foreach ($rirstat_file as &$x) { |
102 | $x = trim($x); |
102 | $x = trim($x); |
103 | if ($x == '') continue; |
103 | if ($x == '') continue; |
104 | 104 | ||
105 | $ary = explode('|', $x); |
105 | $ary = explode('|', $x); |
106 | 106 | ||
107 | # arin|* |asn|* |22244|summary |
107 | # arin|* |asn|* |22244|summary |
108 | # ^0 ^1 ^2 ^3 ^4 ^5 |
108 | # ^0 ^1 ^2 ^3 ^4 ^5 |
109 | if ($ary[5] == 'summary') continue; |
109 | if ($ary[5] == 'summary') continue; |
110 | 110 | ||
111 | # Example: |
111 | # Example: |
112 | # afrinic|ZM|ipv4|196.46.192.0|8192|20031024|allocated |
112 | # afrinic|ZM|ipv4|196.46.192.0|8192|20031024|allocated |
113 | # ^0 ^1 ^2 ^3 ^4 ^5 ^6 |
113 | # ^0 ^1 ^2 ^3 ^4 ^5 ^6 |
114 | 114 | ||
115 | # 2.1AP resp 2.3 (= extended) are compatible with 2.0 |
115 | # 2.1AP resp 2.3 (= extended) are compatible with 2.0 |
116 | 116 | ||
117 | $type = $ary[2]; |
117 | $type = $ary[2]; |
118 | if ($type != 'ipv4') continue; |
118 | if ($type != 'ipv4') continue; |
119 | 119 | ||
120 | $baseip = $ary[3]; |
120 | $baseip = $ary[3]; |
121 | $nums = $ary[4]; |
121 | $nums = $ary[4]; |
122 | # $state = $ary[6]; |
122 | # $state = $ary[6]; |
123 | 123 | ||
124 | $topip = ipv4_add($baseip, $nums-1); |
124 | $topip = ipv4_add($baseip, $nums-1); |
125 | 125 | ||
126 | $rir_blocks[$baseip] = $topip; |
126 | $rir_blocks[$baseip] = $topip; |
127 | } |
127 | } |
128 | } |
128 | } |
129 | 129 | ||
130 | if (USE_IANA_DATA && USE_RIR_STAT_DATA) { |
130 | if (USE_IANA_DATA && USE_RIR_STAT_DATA) { |
131 | # Use both |
131 | # Use both |
132 | $step++; |
132 | $step++; |
133 | echo "IPv4 $rir: Step $step - Merging RIR and IANA stats\n"; |
133 | echo "IPv4 $rir: Step $step - Merging RIR and IANA stats\n"; |
134 | $blocks_merged = ipv4_merge_arrays($rir_blocks, $iana_blocks); |
134 | $blocks_merged = ipv4_merge_arrays($rir_blocks, $iana_blocks); |
135 | } else if (USE_IANA_DATA) { |
135 | } else if (USE_IANA_DATA) { |
136 | # Use only IANA |
136 | # Use only IANA |
137 | $blocks_merged = $iana_blocks; |
137 | $blocks_merged = $iana_blocks; |
138 | } else if (USE_RIR_STAT_DATA) { |
138 | } else if (USE_RIR_STAT_DATA) { |
139 | # Use only RIR data |
139 | # Use only RIR data |
140 | $blocks_merged = $rir_blocks; |
140 | $blocks_merged = $rir_blocks; |
141 | } else { |
141 | } else { |
142 | fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n"); |
142 | fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n"); |
143 | exit(1); |
143 | exit(1); |
144 | } |
144 | } |
145 | 145 | ||
146 | $step++; |
146 | $step++; |
147 | $blocks_merged = ipv4_merge_address_blocks($blocks_merged, "IPv4 $rir: Step $step - "); |
147 | $blocks_merged = ipv4_merge_address_blocks($blocks_merged, "IPv4 $rir: Step $step - "); |
148 | 148 | ||
149 | // --- CIDR finden und ausgeben --- |
149 | // --- CIDR finden und ausgeben --- |
150 | 150 | ||
151 | $step++; |
151 | $step++; |
152 | echo "IPv4 $rir: Step $step - Analyze address ranges and write CIDR blocks\n"; |
152 | echo "IPv4 $rir: Step $step - Analyze address ranges and write CIDR blocks\n"; |
153 | 153 | ||
154 | foreach ($blocks_merged as $baseip => &$topip) { |
154 | foreach ($blocks_merged as $baseip => &$topip) { |
155 | if (INCLUDE_BLOCK_RANGE_COMMENTS) $out .= "# $baseip - $topip\n"; |
155 | if (INCLUDE_BLOCK_RANGE_COMMENTS) $out .= "# $baseip - $topip\n"; |
156 | 156 | ||
157 | $cidrs = ipv4_range2cidr($baseip, $topip, IPV4_SHORT_FORM); |
157 | $cidrs = ipv4_range2cidr($baseip, $topip, IPV4_SHORT_FORM); |
158 | foreach ($cidrs as &$cidr) { |
158 | foreach ($cidrs as &$cidr) { |
159 | $out .= "=$cidr\n"; |
159 | $out .= "=$cidr\n"; |
160 | } |
160 | } |
161 | } |
161 | } |
162 | 162 | ||
163 | # --- Write to file --- |
163 | # --- Write to file --- |
164 | 164 | ||
165 | $step++; |
165 | $step++; |
166 | echo "IPv4 $rir: Step $step - Write to output file\n"; |
166 | echo "IPv4 $rir: Step $step - Write to output file\n"; |
167 | 167 | ||
168 | $h = fopen(PATTERN_DIR."/ipv4_$rir", 'w') or die("Error opening file ".PATTERN_DIR."/ipv4_$rir"); |
168 | $h = fopen(PATTERN_DIR."/ipv4_$rir", 'w') or die("Error opening file ".PATTERN_DIR."/ipv4_$rir"); |
169 | fwrite($h, $out) or die('Could not write to output file'); |
169 | fwrite($h, $out) or die('Could not write to output file'); |
170 | fclose($h); |
170 | fclose($h); |
171 | } |
171 | } |
172 | 172 | ||
173 | echo "IPv4 Finished!\n"; |
173 | echo "IPv4 Finished!\n"; |