WebSVN – logviewer – Diff – /trunk/backend/insert_logs


#!/usr/bin/php
<?php
 
/*
 * ViaThinkSoft LogViewer
 * Copyright 2018-2019 Daniel Marschall, ViaThinkSoft
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
if (php_sapi_name() !== 'cli') {
	die("Error: This script can only be called from command line.\n");
}
 
require_once __DIR__.'/../config.inc.php';
 
$hostname = trim(file_get_contents('/etc/hostname'));
 
if (file_exists(__DIR__."/../db_$hostname.inc.php")) {
	require_once __DIR__."/../db_$hostname.inc.php";
} else {
	require_once __DIR__.'/../db.inc.php';
}
 
$files = array();
foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));
usort($files, create_function('$a,$b', 'return filemtime($a) - filemtime($b);'));
 
$phpfiles = array();
foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));
usort($phpfiles, create_function('$a,$b', 'return filemtime($a) - filemtime($b);'));
 
$file_nr = 0;
$file_max = count($files) + count($phpfiles);
 
// Apache Log Files
 
foreach ($files as $file) {
	$file_nr++;
 
	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
 
	if (substr($file,-3,3) === '.gz') {
		if (IGNORE_GZ) continue;
		$cont = file_get_contents($file);
		$cont = gzdecode($cont);
		if ($cont === false) continue;
		$lines = explode("\n", $cont);
	} else {
		$lines = file($file);
	}
 
	$line_no = 0;
	$line_max = count($lines);
	$logfile = removeLogrotateSuffix($file);
	foreach ($lines as $line) {
		$line_no++;
		$line = trim($line);
 
		if (preg_match('@^\[(.*)\] \[(.*)\] \[(.*)\] \[(.*)\] (.*)$@ismU', $line, $m)) {
			#[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice:  Undefined offset: 11 in /home/d
			$time = $m[1];
			$modul = $m[2];
			$text = $m[5];
 
			$time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);
			$time_mysql = date('Y-m-d H:i:s', strtotime($time));
		} else if (preg_match('@^(.+)\|(.+)\|(.+)\|(.+)$@ismU', $line, $m)) {
			#  5.6 | /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91            | ini              | Ini mbstring.internal_encoding is deprecated.
			// A special implementation of PHP codefixer (showing the full path) . TODO: release
			$time = filemtime($file);
			$modul = 'php_codefixer';
			$text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);
 
			$time_mysql = date('Y-m-d H:i:s', $time);
		} else {
			continue;
		}
 
		$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
		#echo mysql_error();
		if (mysql_num_rows($res) > 0) {
			mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
			            "where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
			#echo mysql_error();
 
		} else {
			mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
			            "values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
			#echo mysql_error();
		}
		echo "file $file_nr/$file_max (line $line_no/$line_max)                     \r";
	}
}
 
// PHP Log files
 
foreach ($phpfiles as $file) {
	$file_nr++;
 
	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
 
	if (substr($file,-3,3) === '.gz') {
		if (IGNORE_GZ) continue;
		$cont = file_get_contents($file);
		$cont = gzdecode($cont);
		if ($cont === false) continue;
	} else {
		$cont = file_get_contents($file);
	}
	$cont = str_replace("\r", "", $cont);
	$cont = str_replace("\n ", " ", $cont);
	$lines = explode("\n", $cont);
 
	$line_no = 0;
	$line_max = count($lines);
	$logfile = removeLogrotateSuffix($file);
	foreach ($lines as $line) {
		$line_no++;
		$line = trim($line);
 
		if (preg_match('@^\[(.*)\] ((.*)(\n ){0,1})$@ismU', $line, $m)) {
			# [19-Aug-2017 23:00:54 europe/berlin] PHP Notice:  Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364
			$time = $m[1];
			$modul = '';
			$text = $m[2];
 
			$time = trim(substr($time, 0, 20));
			$time_mysql = date('Y-m-d H:i:s', strtotime($time));
		} else {
			continue;
		}
 
		$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
		#echo mysql_error();
		if (mysql_num_rows($res) > 0) {
			mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
			            "where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
			#echo mysql_error();
 
		} else {
			mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
			            "values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
			#echo mysql_error();
		}
		echo "file $file_nr/$file_max (line $line_no/$line_max)                     \r";
	}
}
echo "\n";
 
# ---
 
function removeLogrotateSuffix($filename) {
	$filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);
	return $filename;
}
 

Rev 3	Rev 4
1	#!/usr/bin/php	1	#!/usr/bin/php
2	<?php	2	<?php
3		3
-		4	/*
-		5	* ViaThinkSoft LogViewer
-		6	* Copyright 2018-2019 Daniel Marschall, ViaThinkSoft
-		7	*
-		8	* Licensed under the Apache License, Version 2.0 (the "License");
-		9	* you may not use this file except in compliance with the License.
-		10	* You may obtain a copy of the License at
-		11	*
-		12	* http://www.apache.org/licenses/LICENSE-2.0
-		13	*
-		14	* Unless required by applicable law or agreed to in writing, software
-		15	* distributed under the License is distributed on an "AS IS" BASIS,
-		16	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-		17	* See the License for the specific language governing permissions and
-		18	* limitations under the License.
-		19	*/
-		20
4	if (php_sapi_name() !== 'cli') {	21	if (php_sapi_name() !== 'cli') {
5	die("Error: This script can only be called from command line.\n");	22	die("Error: This script can only be called from command line.\n");
6	}	23	}
7		24
8	require_once __DIR__.'/../config.inc.php';	25	require_once __DIR__.'/../config.inc.php';
9		26
10	$hostname = trim(file_get_contents('/etc/hostname'));	27	$hostname = trim(file_get_contents('/etc/hostname'));
11		28
12	if (file_exists(__DIR__."/../db_$hostname.inc.php")) {	29	if (file_exists(__DIR__."/../db_$hostname.inc.php")) {
13	require_once __DIR__."/../db_$hostname.inc.php";	30	require_once __DIR__."/../db_$hostname.inc.php";
14	} else {	31	} else {
15	require_once __DIR__.'/../db.inc.php';	32	require_once __DIR__.'/../db.inc.php';
16	}	33	}
17		34
18	$files = array();	35	$files = array();
19	foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));	36	foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));
20	usort($files, create_function('$a,$b', 'return filemtime($a) - filemtime($b);'));	37	usort($files, create_function('$a,$b', 'return filemtime($a) - filemtime($b);'));
21		38
22	$phpfiles = array();	39	$phpfiles = array();
23	foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));	40	foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));
24	usort($phpfiles, create_function('$a,$b', 'return filemtime($a) - filemtime($b);'));	41	usort($phpfiles, create_function('$a,$b', 'return filemtime($a) - filemtime($b);'));
25		42
26	$file_nr = 0;	43	$file_nr = 0;
27	$file_max = count($files) + count($phpfiles);	44	$file_max = count($files) + count($phpfiles);
28		45
29	// Apache Log Files	46	// Apache Log Files
30		47
31	foreach ($files as $file) {	48	foreach ($files as $file) {
32	$file_nr++;	49	$file_nr++;
33		50
34	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;	51	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
35		52
36	if (substr($file,-3,3) === '.gz') {	53	if (substr($file,-3,3) === '.gz') {
37	if (IGNORE_GZ) continue;	54	if (IGNORE_GZ) continue;
38	$cont = file_get_contents($file);	55	$cont = file_get_contents($file);
39	$cont = gzdecode($cont);	56	$cont = gzdecode($cont);
40	if ($cont === false) continue;	57	if ($cont === false) continue;
41	$lines = explode("\n", $cont);	58	$lines = explode("\n", $cont);
42	} else {	59	} else {
43	$lines = file($file);	60	$lines = file($file);
44	}	61	}
45		62
46	$line_no = 0;	63	$line_no = 0;
47	$line_max = count($lines);	64	$line_max = count($lines);
48	$logfile = removeLogrotateSuffix($file);	65	$logfile = removeLogrotateSuffix($file);
49	foreach ($lines as $line) {	66	foreach ($lines as $line) {
50	$line_no++;	67	$line_no++;
51	$line = trim($line);	68	$line = trim($line);
52		69
53	if (preg_match('@^\[(.)\] \[(.)\] \[(.)\] \[(.)\] (.*)$@ismU', $line, $m)) {	70	if (preg_match('@^\[(.)\] \[(.)\] \[(.)\] \[(.)\] (.*)$@ismU', $line, $m)) {
54	#[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice: Undefined offset: 11 in /home/d	71	#[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice: Undefined offset: 11 in /home/d
55	$time = $m[1];	72	$time = $m[1];
56	$modul = $m[2];	73	$modul = $m[2];
57	$text = $m[5];	74	$text = $m[5];
58		75
59	$time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);	76	$time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);
60	$time_mysql = date('Y-m-d H:i:s', strtotime($time));	77	$time_mysql = date('Y-m-d H:i:s', strtotime($time));
61	} else if (preg_match('@^(.+)\\|(.+)\\|(.+)\\|(.+)$@ismU', $line, $m)) {	78	} else if (preg_match('@^(.+)\\|(.+)\\|(.+)\\|(.+)$@ismU', $line, $m)) {
62	# 5.6 \| /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91 \| ini \| Ini mbstring.internal_encoding is deprecated.	79	# 5.6 \| /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91 \| ini \| Ini mbstring.internal_encoding is deprecated.
63	// A special implementation of PHP codefixer (showing the full path) . TODO: release	80	// A special implementation of PHP codefixer (showing the full path) . TODO: release
64	$time = filemtime($file);	81	$time = filemtime($file);
65	$modul = 'php_codefixer';	82	$modul = 'php_codefixer';
66	$text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);	83	$text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);
67		84
68	$time_mysql = date('Y-m-d H:i:s', $time);	85	$time_mysql = date('Y-m-d H:i:s', $time);
69	} else {	86	} else {
70	continue;	87	continue;
71	}	88	}
72		89
73	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");	90	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
74	#echo mysql_error();	91	#echo mysql_error();
75	if (mysql_num_rows($res) > 0) {	92	if (mysql_num_rows($res) > 0) {
76	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .	93	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
77	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");	94	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
78	#echo mysql_error();	95	#echo mysql_error();
79		96
80	} else {	97	} else {
81	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .	98	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
82	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");	99	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
83	#echo mysql_error();	100	#echo mysql_error();
84	}	101	}
85	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";	102	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";
86	}	103	}
87	}	104	}
88		105
89	// PHP Log files	106	// PHP Log files
90		107
91	foreach ($phpfiles as $file) {	108	foreach ($phpfiles as $file) {
92	$file_nr++;	109	$file_nr++;
93		110
94	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;	111	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
95		112
96	if (substr($file,-3,3) === '.gz') {	113	if (substr($file,-3,3) === '.gz') {
97	if (IGNORE_GZ) continue;	114	if (IGNORE_GZ) continue;
98	$cont = file_get_contents($file);	115	$cont = file_get_contents($file);
99	$cont = gzdecode($cont);	116	$cont = gzdecode($cont);
100	if ($cont === false) continue;	117	if ($cont === false) continue;
101	} else {	118	} else {
102	$cont = file_get_contents($file);	119	$cont = file_get_contents($file);
103	}	120	}
104	$cont = str_replace("\r", "", $cont);	121	$cont = str_replace("\r", "", $cont);
105	$cont = str_replace("\n ", " ", $cont);	122	$cont = str_replace("\n ", " ", $cont);
106	$lines = explode("\n", $cont);	123	$lines = explode("\n", $cont);
107		124
108	$line_no = 0;	125	$line_no = 0;
109	$line_max = count($lines);	126	$line_max = count($lines);
110	$logfile = removeLogrotateSuffix($file);	127	$logfile = removeLogrotateSuffix($file);
111	foreach ($lines as $line) {	128	foreach ($lines as $line) {
112	$line_no++;	129	$line_no++;
113	$line = trim($line);	130	$line = trim($line);
114		131
115	if (preg_match('@^\[(.)\] ((.)(\n ){0,1})$@ismU', $line, $m)) {	132	if (preg_match('@^\[(.)\] ((.)(\n ){0,1})$@ismU', $line, $m)) {
116	# [19-Aug-2017 23:00:54 europe/berlin] PHP Notice: Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364	133	# [19-Aug-2017 23:00:54 europe/berlin] PHP Notice: Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364
117	$time = $m[1];	134	$time = $m[1];
118	$modul = '';	135	$modul = '';
119	$text = $m[2];	136	$text = $m[2];
120		137
121	$time = trim(substr($time, 0, 20));	138	$time = trim(substr($time, 0, 20));
122	$time_mysql = date('Y-m-d H:i:s', strtotime($time));	139	$time_mysql = date('Y-m-d H:i:s', strtotime($time));
123	} else {	140	} else {
124	continue;	141	continue;
125	}	142	}
126		143
127	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");	144	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
128	#echo mysql_error();	145	#echo mysql_error();
129	if (mysql_num_rows($res) > 0) {	146	if (mysql_num_rows($res) > 0) {
130	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .	147	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
131	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");	148	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
132	#echo mysql_error();	149	#echo mysql_error();
133		150
134	} else {	151	} else {
135	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .	152	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
136	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");	153	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
137	#echo mysql_error();	154	#echo mysql_error();
138	}	155	}
139	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";	156	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";
140	}	157	}
141	}	158	}
142	echo "\n";	159	echo "\n";
143		160
144	# ---	161	# ---
145		162
146	function removeLogrotateSuffix($filename) {	163	function removeLogrotateSuffix($filename) {
147	$filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);	164	$filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);
148	return $filename;	165	return $filename;
149	}	166	}
150		167

Subversion Repositories logviewer

logviewer/trunk/backend/insert_logs – Rev 3 → 4