WebSVN – vgwhois – Diff – /trunk/maintenance/pattern-generator/generate_ipv4


#!/usr/bin/php
<?php
 
#
#  VWhois (ViaThinkSoft WHOIS, a fork of generic Whois / gwhois)
#  Maintenance / Developer utilities
#
#  (c) 2012-2019 by Daniel Marschall, ViaThinkSoft <info@daniel-marschall.de>
#
#  License: https://www.gnu.org/licenses/gpl-2.0.html (GPL version 2)
 
 
 
#
 
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED);
 
require_once __DIR__ . '/config.inc.php';
require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
require_once __DIR__ . '/rirs.inc.php';
 
if (USE_IANA_DATA) {
	echo "IPv4 Preparation: Get IANA delegation file\n";
 
	$iana_root_data = explode("\n", cached_file(IANA_IPV4_ASSIGNMENTS, CACHE_FILE_DIR));
	$iana_root_data = array_map('trim', $iana_root_data);
 
	$iana_root_ver  = trim(str_replace('-', '', $iana_root_data[3]));
	$iana_root_data = grep($iana_root_data, 'whois.');
}
 
foreach ($rirs as &$rir) {
	$step = 0;
	$out = '';
 
	$loc_rir_whois_server = $rir_whois_server[$rir];
 
	if (USE_RIR_STAT_DATA) {
		$step++;
		echo "IPv4 $rir: Step $step - Get RIR statistics file\n";
 
		$rirstat_url  = $rirstat_urls[$rir];
		$rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR));
 
		$rirstat_file = antigrep($rirstat_file, '#');
 
		$tmp = explode('|', $rirstat_file[0]);
		$rirstat_ver = $tmp[5]; // enddate
		# $rirstat_ver = substr($rirstat_ver, 0, 4) . '-' . substr($rirstat_ver, 4, 2) . '-' . substr($rirstat_ver, 6, 2);
		unset($tmp);
 
		$out .= "#: version $rirstat_ver\n\n";
	} else if (USE_IANA_DATA) {
		$out .= "#: version $iana_root_ver\n\n";
	} else {
		fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n");
		exit(1);
	}
 
	$out .= "# ".strtoupper($rir)." IPv4\n";
	$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n";
	$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n";
	if (USE_IANA_DATA)     $out .= "# Source: ($iana_root_ver) ".IANA_IPV4_ASSIGNMENTS."\n";
	if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n";
	$out .= "\n";
 
	$out .= ":whois|$loc_rir_whois_server";
	if ($ipv4_additional_params[$rir] != '') {
		$out .= '|prefix='.$ipv4_additional_params[$rir].'|';
	}
	$out .= "\n";
 
	$iana_blocks = array();
	if (USE_IANA_DATA) {
		$step++;
		echo "IPv4 $rir: Step $step - Parse IANA root blocks\n";
 
		$loc_roots = grep($iana_root_data, $loc_rir_whois_server);
 
		$iana_ary = array();
		foreach ($loc_roots as &$lr) {
			$lr = trim($lr);
			$tmp = explode(' ', $lr, 2);
			$lr = $tmp[0];
			unset($tmp);
			$lr = preg_replace('|^0*|s', '', $lr);
 
			// Break up the CIDR
			$ary = explode('/', $lr, 2);
			$bry = ipv4_cidr2range($ary[0], $ary[1]);
			$iana_blocks[$bry[0]] = $bry[1];
		}
	}
 
	$rir_blocks = array();
	if (USE_RIR_STAT_DATA) {
		$step++;
		echo "IPv4 $rir: Step $step - Parse RIR blocks\n";
 
#		$rirstat_file = grep($rirstat_file, 'ipv4');
#		$rirstat_file = antigrep($rirstat_file, 'summary');
##		array_shift($rirstat_file); // header line
 
		foreach ($rirstat_file as &$x) {
			$x = trim($x);
			if ($x == '') continue;
 
			$ary = explode('|', $x);
 
			# arin|* |asn|* |22244|summary
			# ^0   ^1 ^2  ^3 ^4    ^5
			if ($ary[5] == 'summary') continue;
 
			# Example:
			# afrinic|ZM|ipv4|196.46.192.0|8192|20031024|allocated
			# ^0      ^1 ^2   ^3           ^4   ^5       ^6
 
			# 2.1AP resp 2.3 (= extended) are compatible with 2.0
 
			$type = $ary[2];
			if ($type != 'ipv4') continue;
 
			$baseip = $ary[3];
			$nums = $ary[4];
			# $state = $ary[6];
 
			$topip = ipv4_add($baseip, $nums-1);
 
			$rir_blocks[$baseip] = $topip;
		}
	}
 
	if (USE_IANA_DATA && USE_RIR_STAT_DATA) {
		# Use both
		$step++;
		echo "IPv4 $rir: Step $step - Merging RIR and IANA stats\n";
		$blocks_merged = ipv4_merge_arrays($rir_blocks, $iana_blocks);
	} else if (USE_IANA_DATA) {
		# Use only IANA
		$blocks_merged = $iana_blocks;
	} else if (USE_RIR_STAT_DATA) {
		# Use only RIR data
		$blocks_merged = $rir_blocks;
	} else {
		fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n");
		exit(1);
	}
 
	$step++;
	$blocks_merged = ipv4_merge_address_blocks($blocks_merged, "IPv4 $rir: Step $step - ");
 
	// --- CIDR finden und ausgeben ---
 
	$step++;
	echo "IPv4 $rir: Step $step - Analyze address ranges and write CIDR blocks\n";
 
	foreach ($blocks_merged as $baseip => &$topip) {
		if (INCLUDE_BLOCK_RANGE_COMMENTS) $out .= "# $baseip - $topip\n";
 
		$cidrs = ipv4_range2cidr($baseip, $topip, IPV4_SHORT_FORM);
		foreach ($cidrs as &$cidr) {
			$out .= "=$cidr\n";
		}
	}
 
	# --- Write to file ---
 
	$step++;
	echo "IPv4 $rir: Step $step - Write to output file\n";
 
	$h = fopen(PATTERN_DIR."/ipv4_$rir", 'w') or die("Error opening file ".PATTERN_DIR."/ipv4_$rir");
	fwrite($h, $out) or die('Could not write to output file');
	fclose($h);
}
 
echo "IPv4 Finished!\n";

Rev 4	Rev 5
1	#!/usr/bin/php	1	#!/usr/bin/php
2	<?php	2	<?php
3		3
4	#	4	#
5	# generic Whois - Automatic Pattern Generator: IPv4	5	# VWhois (ViaThinkSoft WHOIS, a fork of generic Whois / gwhois)
-		6	# Maintenance / Developer utilities
6	#	7	#
7	# (c) 2012-2013 Daniel Marschall, ViaThinkSoft [www.viathinksoft.de]	8	# (c) 2012-2019 by Daniel Marschall, ViaThinkSoft <info@daniel-marschall.de>
8	#	9	#
9	# Distribution, usage etc. pp. regulated by the current version of GPL.	10	# License: https://www.gnu.org/licenses/gpl-2.0.html (GPL version 2)
10	#	-
11	#	-
12	# Version 2013-03-08	-
13	#	11	#
14		12
15	error_reporting(E_ALL \| E_NOTICE \| E_STRICT \| E_DEPRECATED);	13	error_reporting(E_ALL \| E_NOTICE \| E_STRICT \| E_DEPRECATED);
16		14
17	require_once __DIR__ . '/config.inc.php';	15	require_once __DIR__ . '/config.inc.php';
18	require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';	16	require_once __DIR__ . '/../../shared/php_includes/common_functions.inc.php';
19	require_once __DIR__ . '/rirs.inc.php';	17	require_once __DIR__ . '/rirs.inc.php';
20		18
21	if (USE_IANA_DATA) {	19	if (USE_IANA_DATA) {
22	echo "IPv4 Preparation: Get IANA delegation file\n";	20	echo "IPv4 Preparation: Get IANA delegation file\n";
23		21
24	$iana_root_data = explode("\n", cached_file(IANA_IPV4_ASSIGNMENTS, CACHE_FILE_DIR));	22	$iana_root_data = explode("\n", cached_file(IANA_IPV4_ASSIGNMENTS, CACHE_FILE_DIR));
25	$iana_root_data = array_map('trim', $iana_root_data);	23	$iana_root_data = array_map('trim', $iana_root_data);
26		24
27	$iana_root_ver = trim(str_replace('-', '', $iana_root_data[3]));	25	$iana_root_ver = trim(str_replace('-', '', $iana_root_data[3]));
28	$iana_root_data = grep($iana_root_data, 'whois.');	26	$iana_root_data = grep($iana_root_data, 'whois.');
29	}	27	}
30		28
31	foreach ($rirs as &$rir) {	29	foreach ($rirs as &$rir) {
32	$step = 0;	30	$step = 0;
33	$out = '';	31	$out = '';
34		32
35	$loc_rir_whois_server = $rir_whois_server[$rir];	33	$loc_rir_whois_server = $rir_whois_server[$rir];
36		34
37	if (USE_RIR_STAT_DATA) {	35	if (USE_RIR_STAT_DATA) {
38	$step++;	36	$step++;
39	echo "IPv4 $rir: Step $step - Get RIR statistics file\n";	37	echo "IPv4 $rir: Step $step - Get RIR statistics file\n";
40		38
41	$rirstat_url = $rirstat_urls[$rir];	39	$rirstat_url = $rirstat_urls[$rir];
42	$rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR));	40	$rirstat_file = explode("\n", cached_file($rirstat_url, CACHE_FILE_DIR));
43		41
44	$rirstat_file = antigrep($rirstat_file, '#');	42	$rirstat_file = antigrep($rirstat_file, '#');
45		43
46	$tmp = explode('\|', $rirstat_file[0]);	44	$tmp = explode('\|', $rirstat_file[0]);
47	$rirstat_ver = $tmp[5]; // enddate	45	$rirstat_ver = $tmp[5]; // enddate
48	# $rirstat_ver = substr($rirstat_ver, 0, 4) . '-' . substr($rirstat_ver, 4, 2) . '-' . substr($rirstat_ver, 6, 2);	46	# $rirstat_ver = substr($rirstat_ver, 0, 4) . '-' . substr($rirstat_ver, 4, 2) . '-' . substr($rirstat_ver, 6, 2);
49	unset($tmp);	47	unset($tmp);
50		48
51	$out .= "#: version $rirstat_ver\n\n";	49	$out .= "#: version $rirstat_ver\n\n";
52	} else if (USE_IANA_DATA) {	50	} else if (USE_IANA_DATA) {
53	$out .= "#: version $iana_root_ver\n\n";	51	$out .= "#: version $iana_root_ver\n\n";
54	} else {	52	} else {
55	fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n");	53	fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n");
56	exit(1);	54	exit(1);
57	}	55	}
58		56
59	$out .= "# ".strtoupper($rir)." IPv4\n";	57	$out .= "# ".strtoupper($rir)." IPv4\n";
60	$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n";	58	$out .= "# Automatically generated by ".__DIR__."/gwhois-pattern-update\n";
61	$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n";	59	$out .= "# Generation timestamp: ".date('Y-m-d H:i:s \G\M\TO')."\n";
62	if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) ".IANA_IPV4_ASSIGNMENTS."\n";	60	if (USE_IANA_DATA) $out .= "# Source: ($iana_root_ver) ".IANA_IPV4_ASSIGNMENTS."\n";
63	if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n";	61	if (USE_RIR_STAT_DATA) $out .= "# Source: ($rirstat_ver) $rirstat_url\n";
64	$out .= "\n";	62	$out .= "\n";
65		63
66	$out .= ":whois\|$loc_rir_whois_server";	64	$out .= ":whois\|$loc_rir_whois_server";
67	if ($ipv4_additional_params[$rir] != '') {	65	if ($ipv4_additional_params[$rir] != '') {
68	$out .= '\|prefix='.$ipv4_additional_params[$rir].'\|';	66	$out .= '\|prefix='.$ipv4_additional_params[$rir].'\|';
69	}	67	}
70	$out .= "\n";	68	$out .= "\n";
71		69
72	$iana_blocks = array();	70	$iana_blocks = array();
73	if (USE_IANA_DATA) {	71	if (USE_IANA_DATA) {
74	$step++;	72	$step++;
75	echo "IPv4 $rir: Step $step - Parse IANA root blocks\n";	73	echo "IPv4 $rir: Step $step - Parse IANA root blocks\n";
76		74
77	$loc_roots = grep($iana_root_data, $loc_rir_whois_server);	75	$loc_roots = grep($iana_root_data, $loc_rir_whois_server);
78		76
79	$iana_ary = array();	77	$iana_ary = array();
80	foreach ($loc_roots as &$lr) {	78	foreach ($loc_roots as &$lr) {
81	$lr = trim($lr);	79	$lr = trim($lr);
82	$tmp = explode(' ', $lr, 2);	80	$tmp = explode(' ', $lr, 2);
83	$lr = $tmp[0];	81	$lr = $tmp[0];
84	unset($tmp);	82	unset($tmp);
85	$lr = preg_replace('\|^0*\|s', '', $lr);	83	$lr = preg_replace('\|^0*\|s', '', $lr);
86		84
87	// Break up the CIDR	85	// Break up the CIDR
88	$ary = explode('/', $lr, 2);	86	$ary = explode('/', $lr, 2);
89	$bry = ipv4_cidr2range($ary[0], $ary[1]);	87	$bry = ipv4_cidr2range($ary[0], $ary[1]);
90	$iana_blocks[$bry[0]] = $bry[1];	88	$iana_blocks[$bry[0]] = $bry[1];
91	}	89	}
92	}	90	}
93		91
94	$rir_blocks = array();	92	$rir_blocks = array();
95	if (USE_RIR_STAT_DATA) {	93	if (USE_RIR_STAT_DATA) {
96	$step++;	94	$step++;
97	echo "IPv4 $rir: Step $step - Parse RIR blocks\n";	95	echo "IPv4 $rir: Step $step - Parse RIR blocks\n";
98		96
99	# $rirstat_file = grep($rirstat_file, 'ipv4');	97	# $rirstat_file = grep($rirstat_file, 'ipv4');
100	# $rirstat_file = antigrep($rirstat_file, 'summary');	98	# $rirstat_file = antigrep($rirstat_file, 'summary');
101	## array_shift($rirstat_file); // header line	99	## array_shift($rirstat_file); // header line
102		100
103	foreach ($rirstat_file as &$x) {	101	foreach ($rirstat_file as &$x) {
104	$x = trim($x);	102	$x = trim($x);
105	if ($x == '') continue;	103	if ($x == '') continue;
106		104
107	$ary = explode('\|', $x);	105	$ary = explode('\|', $x);
108		106
109	# arin\|* \|asn\|* \|22244\|summary	107	# arin\|* \|asn\|* \|22244\|summary
110	# ^0 ^1 ^2 ^3 ^4 ^5	108	# ^0 ^1 ^2 ^3 ^4 ^5
111	if ($ary[5] == 'summary') continue;	109	if ($ary[5] == 'summary') continue;
112		110
113	# Example:	111	# Example:
114	# afrinic\|ZM\|ipv4\|196.46.192.0\|8192\|20031024\|allocated	112	# afrinic\|ZM\|ipv4\|196.46.192.0\|8192\|20031024\|allocated
115	# ^0 ^1 ^2 ^3 ^4 ^5 ^6	113	# ^0 ^1 ^2 ^3 ^4 ^5 ^6
116		114
117	# 2.1AP resp 2.3 (= extended) are compatible with 2.0	115	# 2.1AP resp 2.3 (= extended) are compatible with 2.0
118		116
119	$type = $ary[2];	117	$type = $ary[2];
120	if ($type != 'ipv4') continue;	118	if ($type != 'ipv4') continue;
121		119
122	$baseip = $ary[3];	120	$baseip = $ary[3];
123	$nums = $ary[4];	121	$nums = $ary[4];
124	# $state = $ary[6];	122	# $state = $ary[6];
125		123
126	$topip = ipv4_add($baseip, $nums-1);	124	$topip = ipv4_add($baseip, $nums-1);
127		125
128	$rir_blocks[$baseip] = $topip;	126	$rir_blocks[$baseip] = $topip;
129	}	127	}
130	}	128	}
131		129
132	if (USE_IANA_DATA && USE_RIR_STAT_DATA) {	130	if (USE_IANA_DATA && USE_RIR_STAT_DATA) {
133	# Use both	131	# Use both
134	$step++;	132	$step++;
135	echo "IPv4 $rir: Step $step - Merging RIR and IANA stats\n";	133	echo "IPv4 $rir: Step $step - Merging RIR and IANA stats\n";
136	$blocks_merged = ipv4_merge_arrays($rir_blocks, $iana_blocks);	134	$blocks_merged = ipv4_merge_arrays($rir_blocks, $iana_blocks);
137	} else if (USE_IANA_DATA) {	135	} else if (USE_IANA_DATA) {
138	# Use only IANA	136	# Use only IANA
139	$blocks_merged = $iana_blocks;	137	$blocks_merged = $iana_blocks;
140	} else if (USE_RIR_STAT_DATA) {	138	} else if (USE_RIR_STAT_DATA) {
141	# Use only RIR data	139	# Use only RIR data
142	$blocks_merged = $rir_blocks;	140	$blocks_merged = $rir_blocks;
143	} else {	141	} else {
144	fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n");	142	fwrite(STDERR, "Configuration error: USE_RIR_STAT_DATA and USE_IANA_DATA cannot be both false.\n");
145	exit(1);	143	exit(1);
146	}	144	}
147		145
148	$step++;	146	$step++;
149	$blocks_merged = ipv4_merge_address_blocks($blocks_merged, "IPv4 $rir: Step $step - ");	147	$blocks_merged = ipv4_merge_address_blocks($blocks_merged, "IPv4 $rir: Step $step - ");
150		148
151	// --- CIDR finden und ausgeben ---	149	// --- CIDR finden und ausgeben ---
152		150
153	$step++;	151	$step++;
154	echo "IPv4 $rir: Step $step - Analyze address ranges and write CIDR blocks\n";	152	echo "IPv4 $rir: Step $step - Analyze address ranges and write CIDR blocks\n";
155		153
156	foreach ($blocks_merged as $baseip => &$topip) {	154	foreach ($blocks_merged as $baseip => &$topip) {
157	if (INCLUDE_BLOCK_RANGE_COMMENTS) $out .= "# $baseip - $topip\n";	155	if (INCLUDE_BLOCK_RANGE_COMMENTS) $out .= "# $baseip - $topip\n";
158		156
159	$cidrs = ipv4_range2cidr($baseip, $topip, IPV4_SHORT_FORM);	157	$cidrs = ipv4_range2cidr($baseip, $topip, IPV4_SHORT_FORM);
160	foreach ($cidrs as &$cidr) {	158	foreach ($cidrs as &$cidr) {
161	$out .= "=$cidr\n";	159	$out .= "=$cidr\n";
162	}	160	}
163	}	161	}
164		162
165	# --- Write to file ---	163	# --- Write to file ---
166		164
167	$step++;	165	$step++;
168	echo "IPv4 $rir: Step $step - Write to output file\n";	166	echo "IPv4 $rir: Step $step - Write to output file\n";
169		167
170	$h = fopen(PATTERN_DIR."/ipv4_$rir", 'w') or die("Error opening file ".PATTERN_DIR."/ipv4_$rir");	168	$h = fopen(PATTERN_DIR."/ipv4_$rir", 'w') or die("Error opening file ".PATTERN_DIR."/ipv4_$rir");
171	fwrite($h, $out) or die('Could not write to output file');	169	fwrite($h, $out) or die('Could not write to output file');
172	fclose($h);	170	fclose($h);
173	}	171	}
174		172
175	echo "IPv4 Finished!\n";	173	echo "IPv4 Finished!\n";

Subversion Repositories vgwhois

vgwhois/trunk/maintenance/pattern-generator/generate_ipv4 – Rev 4 → 5