WebSVN – logviewer – Diff – /trunk/backend/insert_logs.sh


#!/usr/bin/php
<?php
 
/*
 * ViaThinkSoft LogViewer
 * Copyright 2018-2019 Daniel Marschall, ViaThinkSoft
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
if (php_sapi_name() !== 'cli') {
        die("Error: This script can only be called from command line.\n");
}
 
require_once __DIR__.'/../config.inc.php';
 
$hostname = trim(file_get_contents('/etc/hostname'));
 
if (file_exists(__DIR__."/../db_$hostname.inc.php")) {
        require_once __DIR__."/../db_$hostname.inc.php";
} else {
        require_once __DIR__.'/../db.inc.php';
}
 
$files = array();
foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));
usort($files, function($a,$b) { return filemtime($a) - filemtime($b); });
 
$phpfiles = array();
foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));
usort($phpfiles, function($a,$b) { return filemtime($a) - filemtime($b); });
 
$file_nr = 0;
$file_max = count($files) + count($phpfiles);
 
$TMP_FILE = __DIR__ . '/.insertlogs.cache';
if (file_exists($TMP_FILE)) {
        $cont = file_get_contents($TMP_FILE);
        $cache = unserialize($cont);
} else {
        $cache = array();
}
 
// Apache Log Files
 
foreach ($files as $file) {
        $file_nr++;
 
        if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;
 
        if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
 
        if (substr($file,-3,3) === '.gz') {
                if (IGNORE_GZ) continue;
                $cont = file_get_contents($file);
                $cont = gzdecode($cont);
                if ($cont === false) continue;
                $lines = explode("\n", $cont);
        } else {
                $lines = file($file);
        }
 
        $line_no = 0;
        $line_max = count($lines);
        $logfile = removeLogrotateSuffix($file);
        foreach ($lines as $line) {
                $line_no++;
                $line = trim($line);
 
                if (preg_match('@^\[(.*)\] \[(.*)\] \[(.*)\] \[(.*)\] (.*)$@ismU', $line, $m)) {
                        #[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice:  Undefined offset: 11 in /home/d
                        $time = $m[1];
                        $modul = $m[2];
                        $text = $m[5];
 
                        $time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);
                        $time_mysql = date('Y-m-d H:i:s', strtotime($time));
                } else if (preg_match('@^(.+)\|(.+)\|(.+)\|(.+)$@ismU', $line, $m)) {
                        #  5.6 | /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91            | ini              | Ini mbstring.internal_encoding is deprecated.
                        // A special implementation of PHP codefixer (showing the full path) . TODO: release
                        $time = filemtime($file);
                        $modul = 'php_codefixer';
                        $text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);
 
                        $time_mysql = date('Y-m-d H:i:s', $time);
                } else {
                        continue;
                }
 
                $res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
                #echo mysql_error();
                if (mysql_num_rows($res) > 0) {
                        mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
                                    "where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
                        #echo mysql_error();
 
                } else {
                        mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
                                    "values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
                        #echo mysql_error();
                }
                echo "file $file_nr/$file_max (line $line_no/$line_max)                     \r";
        }
 
        $cache[$file] = md5_file($file);
}
 
// PHP Log files
 
foreach ($phpfiles as $file) {
        $file_nr++;
 
        if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;
 
        if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
 
        if (substr($file,-3,3) === '.gz') {
                if (IGNORE_GZ) continue;
                $cont = file_get_contents($file);
                $cont = gzdecode($cont);
                if ($cont === false) continue;
        } else {
                $cont = file_get_contents($file);
        }
        $cont = str_replace("\r", "", $cont);
        $cont = str_replace("\n ", " ", $cont);
        $lines = explode("\n", $cont);
 
        $line_no = 0;
        $line_max = count($lines);
        $logfile = removeLogrotateSuffix($file);
        foreach ($lines as $line) {
                $line_no++;
                $line = trim($line);
 
                if (preg_match('@^\[(.*)\] ((.*)(\n ){0,1})$@ismU', $line, $m)) {
                        # [19-Aug-2017 23:00:54 europe/berlin] PHP Notice:  Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364
                        $time = $m[1];
                        $modul = '';
                        $text = $m[2];
 
                        $time = trim(substr($time, 0, 20));
                        $time_mysql = date('Y-m-d H:i:s', strtotime($time));
                } else {
                        continue;
                }
 
                $res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
                #echo mysql_error();
                if (mysql_num_rows($res) > 0) {
                        mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
                                    "where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
                        #echo mysql_error();
 
                } else {
                        mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
                                    "values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
                        #echo mysql_error();
                }
                echo "file $file_nr/$file_max (line $line_no/$line_max)                     \r";
        }
 
        $cache[$file] = md5_file($file);
}
echo "\n";
 
file_put_contents($TMP_FILE, serialize($cache));
 
# ---
 
function removeLogrotateSuffix($filename) {
        $filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);
        return $filename;
}
 
 

Rev 5	Rev 7
1	#!/usr/bin/php	1	#!/usr/bin/php
2	<?php	2	<?php
3		3
4	/*	4	/*
5	* ViaThinkSoft LogViewer	5	* ViaThinkSoft LogViewer
6	* Copyright 2018-2019 Daniel Marschall, ViaThinkSoft	6	* Copyright 2018-2019 Daniel Marschall, ViaThinkSoft
7	*	7	*
8	* Licensed under the Apache License, Version 2.0 (the "License");	8	* Licensed under the Apache License, Version 2.0 (the "License");
9	* you may not use this file except in compliance with the License.	9	* you may not use this file except in compliance with the License.
10	* You may obtain a copy of the License at	10	* You may obtain a copy of the License at
11	*	11	*
12	* http://www.apache.org/licenses/LICENSE-2.0	12	* http://www.apache.org/licenses/LICENSE-2.0
13	*	13	*
14	* Unless required by applicable law or agreed to in writing, software	14	* Unless required by applicable law or agreed to in writing, software
15	* distributed under the License is distributed on an "AS IS" BASIS,	15	* distributed under the License is distributed on an "AS IS" BASIS,
16	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.	16	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17	* See the License for the specific language governing permissions and	17	* See the License for the specific language governing permissions and
18	* limitations under the License.	18	* limitations under the License.
19	*/	19	*/
20		20
21	if (php_sapi_name() !== 'cli') {	21	if (php_sapi_name() !== 'cli') {
22	die("Error: This script can only be called from command line.\n");	22	die("Error: This script can only be called from command line.\n");
23	}	23	}
24		24
25	require_once __DIR__.'/../config.inc.php';	25	require_once __DIR__.'/../config.inc.php';
26		26
27	$hostname = trim(file_get_contents('/etc/hostname'));	27	$hostname = trim(file_get_contents('/etc/hostname'));
28		28
29	if (file_exists(__DIR__."/../db_$hostname.inc.php")) {	29	if (file_exists(__DIR__."/../db_$hostname.inc.php")) {
30	require_once __DIR__."/../db_$hostname.inc.php";	30	require_once __DIR__."/../db_$hostname.inc.php";
31	} else {	31	} else {
32	require_once __DIR__.'/../db.inc.php';	32	require_once __DIR__.'/../db.inc.php';
33	}	33	}
34		34
35	$files = array();	35	$files = array();
36	foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));	36	foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));
37	usort($files, function($a,$b) { return filemtime($a) - filemtime($b); });	37	usort($files, function($a,$b) { return filemtime($a) - filemtime($b); });
38		38
39	$phpfiles = array();	39	$phpfiles = array();
40	foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));	40	foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));
41	usort($phpfiles, function($a,$b) { return filemtime($a) - filemtime($b); });	41	usort($phpfiles, function($a,$b) { return filemtime($a) - filemtime($b); });
42		42
43	$file_nr = 0;	43	$file_nr = 0;
44	$file_max = count($files) + count($phpfiles);	44	$file_max = count($files) + count($phpfiles);
45		45
46	$TMP_FILE = '/tmp/insert_logs.cache';	46	$TMP_FILE = __DIR__ . '/.insertlogs.cache';
47	if (file_exists($TMP_FILE)) {	47	if (file_exists($TMP_FILE)) {
48	$cont = file_get_contents($TMP_FILE);	48	$cont = file_get_contents($TMP_FILE);
49	$cache = unserialize($cont);	49	$cache = unserialize($cont);
50	} else {	50	} else {
51	$cache = array();	51	$cache = array();
52	}	52	}
53		53
54	// Apache Log Files	54	// Apache Log Files
55		55
56	foreach ($files as $file) {	56	foreach ($files as $file) {
57	$file_nr++;	57	$file_nr++;
58		58
59	if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;	59	if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;
60		60
61	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;	61	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
62		62
63	if (substr($file,-3,3) === '.gz') {	63	if (substr($file,-3,3) === '.gz') {
64	if (IGNORE_GZ) continue;	64	if (IGNORE_GZ) continue;
65	$cont = file_get_contents($file);	65	$cont = file_get_contents($file);
66	$cont = gzdecode($cont);	66	$cont = gzdecode($cont);
67	if ($cont === false) continue;	67	if ($cont === false) continue;
68	$lines = explode("\n", $cont);	68	$lines = explode("\n", $cont);
69	} else {	69	} else {
70	$lines = file($file);	70	$lines = file($file);
71	}	71	}
72		72
73	$line_no = 0;	73	$line_no = 0;
74	$line_max = count($lines);	74	$line_max = count($lines);
75	$logfile = removeLogrotateSuffix($file);	75	$logfile = removeLogrotateSuffix($file);
76	foreach ($lines as $line) {	76	foreach ($lines as $line) {
77	$line_no++;	77	$line_no++;
78	$line = trim($line);	78	$line = trim($line);
79		79
80	if (preg_match('@^\[(.)\] \[(.)\] \[(.)\] \[(.)\] (.*)$@ismU', $line, $m)) {	80	if (preg_match('@^\[(.)\] \[(.)\] \[(.)\] \[(.)\] (.*)$@ismU', $line, $m)) {
81	#[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice: Undefined offset: 11 in /home/d	81	#[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice: Undefined offset: 11 in /home/d
82	$time = $m[1];	82	$time = $m[1];
83	$modul = $m[2];	83	$modul = $m[2];
84	$text = $m[5];	84	$text = $m[5];
85		85
86	$time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);	86	$time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);
87	$time_mysql = date('Y-m-d H:i:s', strtotime($time));	87	$time_mysql = date('Y-m-d H:i:s', strtotime($time));
88	} else if (preg_match('@^(.+)\\|(.+)\\|(.+)\\|(.+)$@ismU', $line, $m)) {	88	} else if (preg_match('@^(.+)\\|(.+)\\|(.+)\\|(.+)$@ismU', $line, $m)) {
89	# 5.6 \| /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91 \| ini \| Ini mbstring.internal_encoding is deprecated.	89	# 5.6 \| /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91 \| ini \| Ini mbstring.internal_encoding is deprecated.
90	// A special implementation of PHP codefixer (showing the full path) . TODO: release	90	// A special implementation of PHP codefixer (showing the full path) . TODO: release
91	$time = filemtime($file);	91	$time = filemtime($file);
92	$modul = 'php_codefixer';	92	$modul = 'php_codefixer';
93	$text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);	93	$text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);
94		94
95	$time_mysql = date('Y-m-d H:i:s', $time);	95	$time_mysql = date('Y-m-d H:i:s', $time);
96	} else {	96	} else {
97	continue;	97	continue;
98	}	98	}
99		99
100	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");	100	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
101	#echo mysql_error();	101	#echo mysql_error();
102	if (mysql_num_rows($res) > 0) {	102	if (mysql_num_rows($res) > 0) {
103	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .	103	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
104	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");	104	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
105	#echo mysql_error();	105	#echo mysql_error();
106		106
107	} else {	107	} else {
108	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .	108	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
109	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");	109	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
110	#echo mysql_error();	110	#echo mysql_error();
111	}	111	}
112	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";	112	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";
113	}	113	}
114		114
115	$cache[$file] = md5_file($file);	115	$cache[$file] = md5_file($file);
116	}	116	}
117		117
118	// PHP Log files	118	// PHP Log files
119		119
120	foreach ($phpfiles as $file) {	120	foreach ($phpfiles as $file) {
121	$file_nr++;	121	$file_nr++;
122		122
123	if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;	123	if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;
124		124
125	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;	125	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
126		126
127	if (substr($file,-3,3) === '.gz') {	127	if (substr($file,-3,3) === '.gz') {
128	if (IGNORE_GZ) continue;	128	if (IGNORE_GZ) continue;
129	$cont = file_get_contents($file);	129	$cont = file_get_contents($file);
130	$cont = gzdecode($cont);	130	$cont = gzdecode($cont);
131	if ($cont === false) continue;	131	if ($cont === false) continue;
132	} else {	132	} else {
133	$cont = file_get_contents($file);	133	$cont = file_get_contents($file);
134	}	134	}
135	$cont = str_replace("\r", "", $cont);	135	$cont = str_replace("\r", "", $cont);
136	$cont = str_replace("\n ", " ", $cont);	136	$cont = str_replace("\n ", " ", $cont);
137	$lines = explode("\n", $cont);	137	$lines = explode("\n", $cont);
138		138
139	$line_no = 0;	139	$line_no = 0;
140	$line_max = count($lines);	140	$line_max = count($lines);
141	$logfile = removeLogrotateSuffix($file);	141	$logfile = removeLogrotateSuffix($file);
142	foreach ($lines as $line) {	142	foreach ($lines as $line) {
143	$line_no++;	143	$line_no++;
144	$line = trim($line);	144	$line = trim($line);
145		145
146	if (preg_match('@^\[(.)\] ((.)(\n ){0,1})$@ismU', $line, $m)) {	146	if (preg_match('@^\[(.)\] ((.)(\n ){0,1})$@ismU', $line, $m)) {
147	# [19-Aug-2017 23:00:54 europe/berlin] PHP Notice: Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364	147	# [19-Aug-2017 23:00:54 europe/berlin] PHP Notice: Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364
148	$time = $m[1];	148	$time = $m[1];
149	$modul = '';	149	$modul = '';
150	$text = $m[2];	150	$text = $m[2];
151		151
152	$time = trim(substr($time, 0, 20));	152	$time = trim(substr($time, 0, 20));
153	$time_mysql = date('Y-m-d H:i:s', strtotime($time));	153	$time_mysql = date('Y-m-d H:i:s', strtotime($time));
154	} else {	154	} else {
155	continue;	155	continue;
156	}	156	}
157		157
158	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");	158	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
159	#echo mysql_error();	159	#echo mysql_error();
160	if (mysql_num_rows($res) > 0) {	160	if (mysql_num_rows($res) > 0) {
161	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .	161	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
162	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");	162	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
163	#echo mysql_error();	163	#echo mysql_error();
164		164
165	} else {	165	} else {
166	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .	166	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
167	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");	167	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
168	#echo mysql_error();	168	#echo mysql_error();
169	}	169	}
170	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";	170	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";
171	}	171	}
172		172
173	$cache[$file] = md5_file($file);	173	$cache[$file] = md5_file($file);
174	}	174	}
175	echo "\n";	175	echo "\n";
176		176
177	file_put_contents($TMP_FILE, serialize($cache));	177	file_put_contents($TMP_FILE, serialize($cache));
178		178
179	# ---	179	# ---
180		180
181	function removeLogrotateSuffix($filename) {	181	function removeLogrotateSuffix($filename) {
182	$filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);	182	$filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);
183	return $filename;	183	return $filename;
184	}	184	}
185		185
186		186

Subversion Repositories logviewer

logviewer/trunk/backend/insert_logs.sh – Rev 5 → 7