Login | ViewVC Help
View File | Revision Log | Show Annotations | Download File | View Changeset | Root Listing
root/vgwhois/trunk/maintenance/qa-monitor/run
Revision: 29
Committed: Sun Aug 11 08:43:32 2019 UTC (12 months ago) by daniel-marschall
File size: 9462 byte(s)
Log Message:
Pattern update

File Contents

# Content
1 #!/usr/bin/php
2 <?php
3
4 #
5 # VGWhoIs (ViaThinkSoft Global WhoIs, a fork of generic Whois / gwhois)
6 # Maintenance / Developer utilities
7 #
8 # (c) 2012-2019 by Daniel Marschall, ViaThinkSoft <info@daniel-marschall.de>
9 #
10 # License: https://www.gnu.org/licenses/gpl-2.0.html (GPL version 2)
11 #
12
13 # TODO: strikte trennung zwischen pattern entwicklung ( = im sdk enthalten ) und lokaler gwi entwicklung ( = packages, eigene vwi services )
14 # TODO: alles modular machen, vieles cachen
15
16 # TODO: was wenn ein rwi entfernt wird? prüfung notwendig dann!
17 # --> das wird im whois-ping test geprueft
18 # TODO: auch unnoetige excludes (anormalities) in config.inc.php melden
19
20 # TODO pattern.d/tld --> check if all existing and disallocated TLDs are inside
21
22 # TODO: konsolenfarben z.b. grün für qa-audit
23
24 error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED);
25
26 # ---
27
28 define('DIR', realpath(__DIR__));
29
30 require_once DIR . '/../../shared/php_includes/common_functions.inc.php';
31
32 define('CACHE_FILE_DIR', DIR . '/../.cache/web');
33
34 $anormale_whois = array();
35 require DIR . '/config.inc.php';
36
37 $pattern_cont = get_united_pattern();
38
39 $global_status = 0;
40
41 # ---
42
43 echo "* Check if every deleted TLD has a notice ( ".DELETED_TLD_LIST." )\n";
44
45 # does not exist as static file at VTS server
46 $del_tld_cont = explode("\n", cached_file(DELETED_TLD_LIST, CACHE_FILE_DIR));
47
48 $loc_good = true;
49 foreach ($del_tld_cont as $tld) {
50 $tld = trim($tld);
51 if ($tld == '') continue;
52 if ($tld[0] == '#') continue;
53 $tld = strtolower($tld);
54 if (!preg_match('@:notice\\|\\|Disallocated by IANA([^\\n]*)[\\r\\n]+\\\\.'.$tld.'\\$@smU', $pattern_cont, $m)) {
55 $loc_good = false;
56 echo "[ !! ] Does not have a notice beginning with \"Disallocated by IANA\": $tld\n";
57 }
58 }
59
60 if ($loc_good) {
61 echo "[ OK ]\n";
62 } else {
63 $global_status=1;
64 }
65
66 # ---
67
68 echo "* Check if every TLD has a single whois entry\n";
69
70 $tld_cont = explode("\n", cached_file(IANA_TLD_REGISTRY, CACHE_FILE_DIR));
71
72 $loc_good = true;
73 foreach ($tld_cont as $tld) {
74 $tld = trim($tld);
75 if ($tld == '') continue;
76 if ($tld[0] == '#') continue;
77 $tld = strtolower($tld);
78 $regex = '@[\r\n]\s*(?<!#)\s*[^a-zA-Z\r\n#]*\\\\.'.trim($tld).'\\$\s*[\r\n]@isU';
79 $pc = $pattern_cont;
80 $pc = preg_replace('@\\(\\?\\:[a-zA-Z\\|]+\\)@isU', '', $pc); # z.B. \.((?:ac|co|net|or|web|sch|mil|go|my|biz)\.id)$
81 $pc = str_replace('(', '', $pc);
82 $pc = str_replace(')', '', $pc); # wegen regexes, z.b. \.(pn)$
83 # if (strpos($pattern_cont, '\.'.trim($tld)) === false) {
84 # if (strpos(str_replace(')', '', $pattern_cont), '\.'.trim($tld).'$') === false) { # ')' wegen RegEx'es
85 if (!preg_match($regex, $pc)) {
86 $iana_url = sprintf(IANA_TLD_ENTRY, $tld);
87 $iana_cont = cached_file($iana_url, CACHE_FILE_DIR);
88 if (strpos($iana_cont, 'This domain is not present in the root zone at this time.') === false) {
89 $loc_good = false;
90
91 # TODO: hier sollte man evtl schon gucken, ob bei iana.org ein whois-name vorliegt.
92 echo "[ !! ] Not in pattern-file: $tld (see $iana_url )\n";
93 }
94 }
95 }
96
97 if ($loc_good) {
98 echo "[ OK ]\n";
99 } else {
100 $global_status=1;
101 }
102
103 # ---
104
105 echo "* Check if TLDs have the latest Root-Whois-Server URLs ( ".ROOT_WHOIS_SERVER_LIST." )\n";
106
107 # TODO: geht nicht wenn 2 TLDs sich einen :whois teilen
108 # geht auch nicht mit domains mit third level tlds oder regexes usw
109
110 # TODO: allerdings wird keine kontrolle gemacht ob die auskommentierte Zeile wirklich zu $tld gehört!
111 echo "Note: For following TLDs, the root whois server may be commented out: ".implode(', ', $anormale_whois)."\n";
112
113 $loc_good = true;
114 $rws_cont = rws_get_list();
115 foreach ($rws_cont as $tld => $whois) {
116 $c = "$tld => $whois";
117
118 $g = false;
119 if (in_arrayi($tld, $anormale_whois)) {
120 # Sonderfälle... whois server die nicht gehen...
121 # Der Server muss dann nur irgendwo vorkommen, in Form einer auskommentierten Zeile
122 $g = (
123 (stripos($pattern_cont, ':whois|'.$whois."\n") !== false) ||
124 (stripos($pattern_cont, 'whois::'.$whois.":::") !== false) || // multiple
125 (stripos($pattern_cont, 'whois::'.$whois."\n") !== false) || // multiple
126 (stripos($pattern_cont, ':inicwhois|'.$whois."\n") !== false) ||
127 (stripos($pattern_cont, 'inicwhois::'.$whois.":::") !== false) || // multiple
128 (stripos($pattern_cont, 'inicwhois::'.$whois."\n") !== false) // multiple
129 );
130 } else {
131 # TODO: auch auskommentierte einträge erlauben, falls die RWI outdated ist
132 $regex = "@\\n:(inic){0,1}whois\\|".preg_quote($whois, '@')."(\\|.*)*\\n(#.*\n)*\\\\.".preg_quote($tld, '@')."\\$@ismU";
133 $g = preg_match($regex, $pattern_cont);
134
135 if (!$g) {
136 # Multiple?
137 # TODO: bessere pruefung auf auskommentiere eintraege
138 $regex = "@(:multiple\\|\\||:::)(inic){0,1}whois::".preg_quote($whois, '@')."(:[^\n]*|)\s*\n(#.*\n)*(\\(\\.\\*\\)){0,1}\\\\.".preg_quote($tld, '@')."\\$@ismU";
139 $g = preg_match($regex, $pattern_cont);
140 }
141 }
142
143 if (!$g) {
144 # If the data is not live, we need to check if the whois server is still official (and not changed since then)
145 if ((!rws_is_live()) && (iana_get_whois($tld) != $whois)) continue;
146
147 # If we reported it as dead, we trust that. (In case it is wrong, the whois-ping will warn about it)
148 if (reported_dead($whois)) continue;
149
150 # Whois servers which are official but are down, are not interesting. Do not warn.
151 if (!gwitc_is_port_open($whois, 43)) continue;
152
153 $loc_good = false;
154 echo "[ !! ] Not in TLD file: $c\n";
155 }
156 unset($g);
157 }
158
159 if ($loc_good) {
160 echo "[ OK ]\n";
161 } else {
162 $global_status=1;
163 }
164
165 # ---
166
167 $own_debian = '20120626.1.2';
168 $own_github = 4; // Latest commit f712050 on 30 Nov 2018
169
170 echo "* Check if there is a newer official version of gwhois which needs to be merged with the ViaThinkSoft fork\n";
171
172 $their_debian = str_replace('-', '.', getLatestGWIversion()); // they use the reserved '-' character in their versions (used for delimiting upstream versions)
173 $their_github = github_commit_count('julijane', 'gwhois');
174
175 $loc_good = true;
176
177 if ($own_debian != $their_debian) {
178 echo '[ !! ] Their DEBIAN version: '.$their_debian." (Our version is based on: ".own_debian.")\n";
179 $loc_good = false;
180 }
181
182 if ($own_github != $their_github) {
183 echo '[ !! ] Their GITHUB version: '.$their_github." (Our version is based on: ".$own_github.")\n";
184 $loc_good = false;
185 }
186
187 if ($loc_good) {
188 echo "[ OK ]\n";
189 } else {
190 $global_status=1;
191 }
192
193 # ---
194
195 echo "* Check if IANA has (probably) a TLD whois list in ".IANA_TLD_FOLDER."\n";
196
197 $x = cached_file(IANA_TLD_FOLDER, CACHE_FILE_DIR);
198 $loc_good = substr_count($x, '.txt">') == 1;
199
200 if ($loc_good) {
201 echo "[ OK ]\n";
202 } else {
203 $global_status=1;
204 echo "[ !! ] Something has changed ! Please change rws_*() in the QA tool!\n";
205 }
206
207 # ---
208
209 echo "* Check if testcases require attention\n";
210
211 $loc_good = true;
212
213 # First check tesetcases in background, if necessary
214
215 $out = array();
216 exec(DIR . "/testcases/batch --mode b", $out, $ec);
217
218 if ($ec > 2) {
219 $loc_good = false;
220 echo "[ !! ] testcases/batch error code $ec\n";
221 echo trim(implode("\n", $out))."\n";
222 }
223
224 # Now check if testcases require attention
225
226 $out = array();
227 exec(DIR . '/testcases/status_short', $out, $ec);
228
229 if ($ec != 0) {
230 $loc_good = false;
231 echo trim(implode("\n", $out))."\n";
232 }
233
234 # Status?
235
236 if ($loc_good) {
237 echo "[ OK ]\n";
238 } else {
239 $global_status=1;
240 }
241
242 # ---
243
244 echo "* Check the status of whois-ping\n";
245
246 $loc_good = true;
247
248 # First ping the servers
249
250 $out = array();
251 exec(DIR . "/whois-ping/whoisping", $out, $ec);
252
253 if ($ec != 0) {
254 $loc_good = false;
255 echo "[ !! ] whois-ping/whoisping error code $ec\n";
256 }
257
258 # Now show errors (but only if a time treshold is exceeded)
259
260 $out = array();
261 exec(DIR . '/whois-ping/showerrors', $out, $ec);
262
263 if ($ec != 0) {
264 $loc_good = false;
265 echo trim(implode("\n", $out))."\n";
266 }
267
268 # Status?
269
270 if ($loc_good) {
271 echo "[ OK ]\n";
272 } else {
273 $global_status=1;
274 }
275
276 # ---
277
278 echo "* Check syntax of all script files (Perl, PHP)\n";
279
280 $out = array();
281 exec(DIR . '/syntax/global-syntax-check', $out, $ec);
282
283 $loc_good = $ec == 0;
284
285 if ($loc_good) {
286 echo "[ OK ]\n";
287 } else {
288 $global_status=1;
289 echo trim(implode("\n", $out))."\n";
290 }
291
292 # ---
293
294 exit($global_status);
295
296 # ---
297
298 function getLatestGWIversion() {
299 $x = cached_file('http://packages.debian.org/search?searchon=names&keywords=gwhois', CACHE_FILE_DIR);
300 preg_match_all('@<br>(.*): all@isU', $x, $m);
301 $m = $m[1];
302 rsort($m);
303 return $m[0]; // biggest version
304 }
305
306 // RWS = Root Whois Server List functions
307 // (needs to be updated as soon as IANA implements this service.)
308 function rws_get_list() {
309 $result = array();
310
311 $tld_cont = explode("\n", cached_file(ROOT_WHOIS_SERVER_LIST, CACHE_FILE_DIR));
312
313 foreach ($tld_cont as &$c) {
314 $c = trim($c);
315 if ($c == '') continue;
316 if ($c[0] == '#') continue;
317 $ary = explode(" => ", $c);
318 $tld = $ary[0];
319 $whois = trim($ary[1]);
320
321 $result[$tld] = $whois;
322 }
323
324 return $result;
325 }
326
327 function rws_is_live() {
328 // The data is collected every week.
329 return false;
330 }
331
332 function iana_get_whois($tld) {
333 $out = QueryWhoisServer('whois.iana.org', $tld);
334
335 if (strpos($out, 'domain:') === false) return false;
336 if (!preg_match("@\nwhois:\\s*([^\\s]+)\n@ismU", $out, $m)) return false;
337 return $m[1];
338 }
339
340 function in_arrayi($needle, $haystack) {
341 return in_array(strtolower($needle), array_map('strtolower', $haystack));
342 }
343
344 function reported_dead($whois) {
345 $dead_servers_raw = file(DIR . '/../config/dead-servers.list');
346 foreach ($dead_servers_raw as &$server) {
347 $server = trim($server);
348 if ($server == '') continue;
349 if ($server[0] == '#') continue;
350 if ($server == $whois) return true;
351 }
352 return false;
353 }

Properties

Name Value
svn:executable *