WebSVN – logviewer – Diff – /trunk/backend/insert_logs.sh


#!/usr/bin/php
<?php
 
/*
 * ViaThinkSoft LogViewer
 * Copyright 2018-2022 Daniel Marschall, ViaThinkSoft
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
if (php_sapi_name() !== 'cli') {
        die("Error: This script can only be called from command line.\n");
}
 
$hostname = trim(file_get_contents('/etc/hostname'));
 
if (file_exists(__DIR__."/../config_$hostname.inc.php")) {
        require_once __DIR__."/../config_$hostname.inc.php";
} else {
        require_once __DIR__.'/../config.inc.php';
}
 
if (file_exists(__DIR__."/../db_$hostname.inc.php")) {
        require_once __DIR__."/../db_$hostname.inc.php";
} else {
        require_once __DIR__.'/../db.inc.php';
}
 
$files = array();
foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));
usort($files, function($a,$b) { return filemtime($a) - filemtime($b); });
 
$phpfiles = array();
foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));
usort($phpfiles, function($a,$b) { return filemtime($a) - filemtime($b); });
 
$file_nr = 0;
$file_max = count($files) + count($phpfiles);
 
$TMP_FILE = __DIR__ . '/.insertlogs.cache';
if (file_exists($TMP_FILE)) {
        $cont = file_get_contents($TMP_FILE);
        $cache = unserialize($cont);
} else {
        $cache = array();
}
 
// Apache Log Files
 
foreach ($files as $file) {
        $file_nr++;
 
        if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;
 
        if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
 
        if (substr($file,-3,3) === '.gz') {
                if (IGNORE_GZ) continue;
                $cont = file_get_contents($file);
                $cont = gzdecode($cont);
                if ($cont === false) continue;
                $lines = explode("\n", $cont);
        } else {
                $lines = file($file);
        }
 
        $line_no = 0;
        $line_max = count($lines);
        $logfile = removeLogrotateSuffix($file);
        foreach ($lines as $line) {
                $line_no++;
                $line = trim($line);
 
                if (preg_match('@^\[(.*)\] \[(.*)\] \[(.*)\] \[(.*)\] (.*)$@ismU', $line, $m)) {
                        #[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice:  Undefined offset: 11 in /home/d
                        $time = $m[1];
                        $modul = $m[2];
                        $text = $m[5];
 
                        $time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);
                        $time_mysql = date('Y-m-d H:i:s', strtotime($time));
                } else if (preg_match('@^(.+)\|(.+)\|(.+)\|(.+)$@ismU', $line, $m)) {
                        #  5.6 | /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91            | ini              | Ini mbstring.internal_encoding is deprecated.
                        // A special implementation of PHP codefixer (showing the full path) . TODO: release
                        $time = filemtime($file);
                        $modul = 'php_codefixer';
                        $text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);
 
                        $time_mysql = date('Y-m-d H:i:s', $time);
                } else {
                        continue;
                }
 
                $res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
                #echo mysql_error();
                if (mysql_num_rows($res) > 0) {
                        mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
                                    "where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
                        #echo mysql_error();
 
                } else {
                        mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
                                    "values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
                        #echo mysql_error();
                }
                echo "file $file_nr/$file_max (line $line_no/$line_max)                     \r";
        }
 
        $cache[$file] = md5_file($file);
}
 
// PHP Log files
 
foreach ($phpfiles as $file) {
        $file_nr++;
 
        if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;
 
        if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
 
        if (substr($file,-3,3) === '.gz') {
                if (IGNORE_GZ) continue;
                $cont = file_get_contents($file);
                $cont = gzdecode($cont);
                if ($cont === false) continue;
        } else {
                $cont = file_get_contents($file);
        }
        $cont = str_replace("\r", "", $cont);
        $cont = str_replace("\n ", " ", $cont);
        $lines = explode("\n", $cont);
 
        $line_no = 0;
        $line_max = count($lines);
        $logfile = removeLogrotateSuffix($file);
        foreach ($lines as $line) {
                $line_no++;
                $line = trim($line);
 
                echo "file $file_nr/$file_max (line $line_no/$line_max)                     \r";
 
                if (preg_match('@^\[(.*)\] ((.*)(\n ){0,1})$@ismU', $line, $m)) {
                        # [19-Aug-2017 23:00:54 europe/berlin] PHP Notice:  Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364
                        $time = $m[1];
                        $modul = '';
                        $text = $m[2];
 
                        $time = trim(substr($time, 0, 20));
                        $time_mysql = date('Y-m-d H:i:s', strtotime($time));
                } else {
                        continue;
                }
 
                $res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
                #echo mysql_error();
                if (mysql_num_rows($res) > 0) {
                        mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
                                    "where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
                        #echo mysql_error();
 
                } else {
                        mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
                                    "values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
                        #echo mysql_error();
                }
        }
 
        $cache[$file] = md5_file($file);
}
echo "\n";
 
file_put_contents($TMP_FILE, serialize($cache));
 
# ---
 
function removeLogrotateSuffix($filename) {
        $filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);
        return $filename;
}
 
 

Rev 8	Rev 9
1	#!/usr/bin/php	1	#!/usr/bin/php
2	<?php	2	<?php
3		3
4	/*	4	/*
5	* ViaThinkSoft LogViewer	5	* ViaThinkSoft LogViewer
6	* Copyright 2018-2019 Daniel Marschall, ViaThinkSoft	6	* Copyright 2018-2022 Daniel Marschall, ViaThinkSoft
7	*	7	*
8	* Licensed under the Apache License, Version 2.0 (the "License");	8	* Licensed under the Apache License, Version 2.0 (the "License");
9	* you may not use this file except in compliance with the License.	9	* you may not use this file except in compliance with the License.
10	* You may obtain a copy of the License at	10	* You may obtain a copy of the License at
11	*	11	*
12	* http://www.apache.org/licenses/LICENSE-2.0	12	* http://www.apache.org/licenses/LICENSE-2.0
13	*	13	*
14	* Unless required by applicable law or agreed to in writing, software	14	* Unless required by applicable law or agreed to in writing, software
15	* distributed under the License is distributed on an "AS IS" BASIS,	15	* distributed under the License is distributed on an "AS IS" BASIS,
16	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.	16	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17	* See the License for the specific language governing permissions and	17	* See the License for the specific language governing permissions and
18	* limitations under the License.	18	* limitations under the License.
19	*/	19	*/
20		20
21	if (php_sapi_name() !== 'cli') {	21	if (php_sapi_name() !== 'cli') {
22	die("Error: This script can only be called from command line.\n");	22	die("Error: This script can only be called from command line.\n");
23	}	23	}
24		24
25	$hostname = trim(file_get_contents('/etc/hostname'));	25	$hostname = trim(file_get_contents('/etc/hostname'));
26		26
27	if (file_exists(__DIR__."/config_$hostname.inc.php")) {	27	if (file_exists(__DIR__."/../config_$hostname.inc.php")) {
28	require_once __DIR__."/config_$hostname.inc.php";	28	require_once __DIR__."/../config_$hostname.inc.php";
29	} else {	29	} else {
30	require_once __DIR__.'/config.inc.php';	30	require_once __DIR__.'/../config.inc.php';
31	}	31	}
32		32
33	if (file_exists(__DIR__."/../db_$hostname.inc.php")) {	33	if (file_exists(__DIR__."/../db_$hostname.inc.php")) {
34	require_once __DIR__."/../db_$hostname.inc.php";	34	require_once __DIR__."/../db_$hostname.inc.php";
35	} else {	35	} else {
36	require_once __DIR__.'/../db.inc.php';	36	require_once __DIR__.'/../db.inc.php';
37	}	37	}
38		38
39	$files = array();	39	$files = array();
40	foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));	40	foreach (apache_log_locations as $tpl) $files = array_merge($files, glob($tpl));
41	usort($files, function($a,$b) { return filemtime($a) - filemtime($b); });	41	usort($files, function($a,$b) { return filemtime($a) - filemtime($b); });
42		42
43	$phpfiles = array();	43	$phpfiles = array();
44	foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));	44	foreach (php_log_locations as $tpl) $phpfiles = array_merge($phpfiles, glob($tpl));
45	usort($phpfiles, function($a,$b) { return filemtime($a) - filemtime($b); });	45	usort($phpfiles, function($a,$b) { return filemtime($a) - filemtime($b); });
46		46
47	$file_nr = 0;	47	$file_nr = 0;
48	$file_max = count($files) + count($phpfiles);	48	$file_max = count($files) + count($phpfiles);
49		49
50	$TMP_FILE = __DIR__ . '/.insertlogs.cache';	50	$TMP_FILE = __DIR__ . '/.insertlogs.cache';
51	if (file_exists($TMP_FILE)) {	51	if (file_exists($TMP_FILE)) {
52	$cont = file_get_contents($TMP_FILE);	52	$cont = file_get_contents($TMP_FILE);
53	$cache = unserialize($cont);	53	$cache = unserialize($cont);
54	} else {	54	} else {
55	$cache = array();	55	$cache = array();
56	}	56	}
57		57
58	// Apache Log Files	58	// Apache Log Files
59		59
60	foreach ($files as $file) {	60	foreach ($files as $file) {
61	$file_nr++;	61	$file_nr++;
62		62
63	if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;	63	if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;
64		64
65	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;	65	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
66		66
67	if (substr($file,-3,3) === '.gz') {	67	if (substr($file,-3,3) === '.gz') {
68	if (IGNORE_GZ) continue;	68	if (IGNORE_GZ) continue;
69	$cont = file_get_contents($file);	69	$cont = file_get_contents($file);
70	$cont = gzdecode($cont);	70	$cont = gzdecode($cont);
71	if ($cont === false) continue;	71	if ($cont === false) continue;
72	$lines = explode("\n", $cont);	72	$lines = explode("\n", $cont);
73	} else {	73	} else {
74	$lines = file($file);	74	$lines = file($file);
75	}	75	}
76		76
77	$line_no = 0;	77	$line_no = 0;
78	$line_max = count($lines);	78	$line_max = count($lines);
79	$logfile = removeLogrotateSuffix($file);	79	$logfile = removeLogrotateSuffix($file);
80	foreach ($lines as $line) {	80	foreach ($lines as $line) {
81	$line_no++;	81	$line_no++;
82	$line = trim($line);	82	$line = trim($line);
83		83
84	if (preg_match('@^\[(.)\] \[(.)\] \[(.)\] \[(.)\] (.*)$@ismU', $line, $m)) {	84	if (preg_match('@^\[(.)\] \[(.)\] \[(.)\] \[(.)\] (.*)$@ismU', $line, $m)) {
85	#[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice: Undefined offset: 11 in /home/d	85	#[Sun Aug 13 15:54:16.054530 2017] [fcgid:warn] [pid 28401] [client 104.236.113.44:52188] mod_fcgid: stderr: PHP Notice: Undefined offset: 11 in /home/d
86	$time = $m[1];	86	$time = $m[1];
87	$modul = $m[2];	87	$modul = $m[2];
88	$text = $m[5];	88	$text = $m[5];
89		89
90	$time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);	90	$time = trim(substr($time, 4, 6)).' '.substr($time, -4).' '.substr($time, 11, 8);
91	$time_mysql = date('Y-m-d H:i:s', strtotime($time));	91	$time_mysql = date('Y-m-d H:i:s', strtotime($time));
92	} else if (preg_match('@^(.+)\\|(.+)\\|(.+)\\|(.+)$@ismU', $line, $m)) {	92	} else if (preg_match('@^(.+)\\|(.+)\\|(.+)\\|(.+)$@ismU', $line, $m)) {
93	# 5.6 \| /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91 \| ini \| Ini mbstring.internal_encoding is deprecated.	93	# 5.6 \| /daten/homes/daniel-marschall/hosts/dracenmarx/public_html/wiki/vendor/oyejorge/less.php/lib:91 \| ini \| Ini mbstring.internal_encoding is deprecated.
94	// A special implementation of PHP codefixer (showing the full path) . TODO: release	94	// A special implementation of PHP codefixer (showing the full path) . TODO: release
95	$time = filemtime($file);	95	$time = filemtime($file);
96	$modul = 'php_codefixer';	96	$modul = 'php_codefixer';
97	$text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);	97	$text = 'PHP Codefixer: ' . trim($m[4]) . ' in ' . trim($m[2]);
98		98
99	$time_mysql = date('Y-m-d H:i:s', $time);	99	$time_mysql = date('Y-m-d H:i:s', $time);
100	} else {	100	} else {
101	continue;	101	continue;
102	}	102	}
103		103
104	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");	104	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
105	#echo mysql_error();	105	#echo mysql_error();
106	if (mysql_num_rows($res) > 0) {	106	if (mysql_num_rows($res) > 0) {
107	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .	107	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
108	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");	108	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
109	#echo mysql_error();	109	#echo mysql_error();
110		110
111	} else {	111	} else {
112	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .	112	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
113	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");	113	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
114	#echo mysql_error();	114	#echo mysql_error();
115	}	115	}
116	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";	116	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";
117	}	117	}
118		118
119	$cache[$file] = md5_file($file);	119	$cache[$file] = md5_file($file);
120	}	120	}
121		121
122	// PHP Log files	122	// PHP Log files
123		123
124	foreach ($phpfiles as $file) {	124	foreach ($phpfiles as $file) {
125	$file_nr++;	125	$file_nr++;
126		126
127	if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;	127	if (isset($cache[$file]) && ($cache[$file] == md5_file($file))) continue;
128		128
129	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;	129	if (time()-filemtime($file) > MAX_DAYS_LOGFILE * 3600) continue;
130		130
131	if (substr($file,-3,3) === '.gz') {	131	if (substr($file,-3,3) === '.gz') {
132	if (IGNORE_GZ) continue;	132	if (IGNORE_GZ) continue;
133	$cont = file_get_contents($file);	133	$cont = file_get_contents($file);
134	$cont = gzdecode($cont);	134	$cont = gzdecode($cont);
135	if ($cont === false) continue;	135	if ($cont === false) continue;
136	} else {	136	} else {
137	$cont = file_get_contents($file);	137	$cont = file_get_contents($file);
138	}	138	}
139	$cont = str_replace("\r", "", $cont);	139	$cont = str_replace("\r", "", $cont);
140	$cont = str_replace("\n ", " ", $cont);	140	$cont = str_replace("\n ", " ", $cont);
141	$lines = explode("\n", $cont);	141	$lines = explode("\n", $cont);
142		142
143	$line_no = 0;	143	$line_no = 0;
144	$line_max = count($lines);	144	$line_max = count($lines);
145	$logfile = removeLogrotateSuffix($file);	145	$logfile = removeLogrotateSuffix($file);
146	foreach ($lines as $line) {	146	foreach ($lines as $line) {
147	$line_no++;	147	$line_no++;
148	$line = trim($line);	148	$line = trim($line);
149		149
150	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";	150	echo "file $file_nr/$file_max (line $line_no/$line_max) \r";
151		151
152	if (preg_match('@^\[(.)\] ((.)(\n ){0,1})$@ismU', $line, $m)) {	152	if (preg_match('@^\[(.)\] ((.)(\n ){0,1})$@ismU', $line, $m)) {
153	# [19-Aug-2017 23:00:54 europe/berlin] PHP Notice: Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364	153	# [19-Aug-2017 23:00:54 europe/berlin] PHP Notice: Undefined variable: ssl in /home/viathinksoft/public_html/serverinfo/index.php on line 364
154	$time = $m[1];	154	$time = $m[1];
155	$modul = '';	155	$modul = '';
156	$text = $m[2];	156	$text = $m[2];
157		157
158	$time = trim(substr($time, 0, 20));	158	$time = trim(substr($time, 0, 20));
159	$time_mysql = date('Y-m-d H:i:s', strtotime($time));	159	$time_mysql = date('Y-m-d H:i:s', strtotime($time));
160	} else {	160	} else {
161	continue;	161	continue;
162	}	162	}
163		163
164	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");	164	$res = mysql_query("select * from vts_fehlerlog where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."';");
165	#echo mysql_error();	165	#echo mysql_error();
166	if (mysql_num_rows($res) > 0) {	166	if (mysql_num_rows($res) > 0) {
167	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .	167	mysql_query("update vts_fehlerlog set anzahl = anzahl + 1, letzter = '$time_mysql' " .
168	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");	168	"where modul = '".mysql_real_escape_string($modul)."' and logfile = '".mysql_real_escape_string($logfile)."' and text = '".mysql_real_escape_string($text)."' and letzter < '".$time_mysql."';");
169	#echo mysql_error();	169	#echo mysql_error();
170		170
171	} else {	171	} else {
172	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .	172	mysql_query("insert into vts_fehlerlog (modul, text, anzahl, letzter, logfile) " .
173	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");	173	"values ('".mysql_real_escape_string($modul)."', '".mysql_real_escape_string($text)."', 1, '".$time_mysql."', '".mysql_real_escape_string($logfile)."');");
174	#echo mysql_error();	174	#echo mysql_error();
175	}	175	}
176	}	176	}
177		177
178	$cache[$file] = md5_file($file);	178	$cache[$file] = md5_file($file);
179	}	179	}
180	echo "\n";	180	echo "\n";
181		181
182	file_put_contents($TMP_FILE, serialize($cache));	182	file_put_contents($TMP_FILE, serialize($cache));
183		183
184	# ---	184	# ---
185		185
186	function removeLogrotateSuffix($filename) {	186	function removeLogrotateSuffix($filename) {
187	$filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);	187	$filename = preg_replace('@\\.(\\d+)(\\.gz){0,1}$@ismU', '', $filename);
188	return $filename;	188	return $filename;
189	}	189	}
190		190
191		191

Subversion Repositories logviewer

logviewer/trunk/backend/insert_logs.sh – Rev 8 → 9