ulfgebhardt/system
1
0
Fork 0
mirror of https://github.com/ulfgebhardt/system.git synced 2025-12-13 09:35:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

cron extraction fixed for pg and huge dataload

Browse Source
This commit is contained in:
Ulf Gebhardt 2015-01-29 17:25:53 +01:00
parent 9dd275911b
commit 6ccda3fed3
3 changed files with 20 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
cron/cron_log2sqlite.php
View File

@ -9,11 +9,9 @@ class cron_log2sqlite extends \SYSTEM\CRON\cronjob{
if( $oldest['year'] >= $now_year && if( $oldest['year'] >= $now_year &&
$oldest['month'] >= $now_month){ $oldest['month'] >= $now_month){
return cronstatus::CRON_STATUS_SUCCESFULLY;} return cronstatus::CRON_STATUS_SUCCESFULLY;}
$filename = \SYSTEM\CONFIG\config::get(\SYSTEM\CONFIG\config_ids::SYS_CRON_LOG2SQLITE_PATH).$oldest['year'].'.'.$oldest['month'].'.db'; $filename = \SYSTEM\CONFIG\config::get(\SYSTEM\CONFIG\config_ids::SYS_CRON_LOG2SQLITE_PATH).$oldest['year'].'.'.$oldest['month'].'.db';
//extract whole month to file //extract whole month to file
$con = new \SYSTEM\DB\Connection(new \SYSTEM\DB\DBInfoSQLite($filename)); $con = new \SYSTEM\DB\Connection(new \SYSTEM\DB\DBInfoSQLite($filename));
//create table //create table
$con->query('CREATE TABLE IF NOT EXISTS `system_log` ('. $con->query('CREATE TABLE IF NOT EXISTS `system_log` ('.
' `ID` INT(10) NOT NULL,'. ' `ID` INT(10) NOT NULL,'.
@ -38,43 +36,32 @@ class cron_log2sqlite extends \SYSTEM\CRON\cronjob{
//write data as trasaction //write data as trasaction
$con->exec('begin transaction'); $con->exec('begin transaction');
set_time_limit(30);
$res = \SYSTEM\DBD\SYS_LOG_MONTH::QQ(array($oldest['month'],$oldest['year'])); $res = \SYSTEM\DBD\SYS_LOG_MONTH::QQ(array($oldest['month'],$oldest['year']));
$i = 0;
$j = 0;
while($row = $res->next()){ while($row = $res->next()){
set_time_limit(30); set_time_limit(30);
$i++; $j++; $row['time'] = array_key_exists('time_pg', $row) ? date("Y-m-d H:i:s", $row['time_pg']) : $row['time'];
$row['time'] = array_key_exists('time_pg', $row) ? $row['time_pg'] : '\''.$row['time'].'\'';
if(!$con->exec('INSERT OR IGNORE INTO '.\SYSTEM\DBD\system_log::NAME_MYS. if(!$con->exec('INSERT OR IGNORE INTO '.\SYSTEM\DBD\system_log::NAME_MYS.
'(`ID`, `class`, `message`, `code`, `file`, `line`, `trace`, `ip`, `querytime`, `time`,'. '(`ID`, `class`, `message`, `code`, `file`, `line`, `trace`, `ip`, `querytime`, `time`,'.
' `server_name`, `server_port`, `request_uri`, `post`,'. ' `server_name`, `server_port`, `request_uri`, `post`,'.
' `http_referer`, `http_user_agent`, `user`, `thrown`)'. ' `http_referer`, `http_user_agent`, `user`, `thrown`)'.
'VALUES ('.$row['ID'].', \''.\SQLite3::escapeString($row['class']).'\', \''.\SQLite3::escapeString($row['message']).'\', '. 'VALUES ('.$row['ID'].', \''.\SQLite3::escapeString($row['class']).'\', \''.\SQLite3::escapeString($row['message']).'\', '.
$row['code'].', \''.\SQLite3::escapeString($row['file']).'\', '.$row['line'].', \''.\SQLite3::escapeString($row['trace']).'\', \''. $row['code'].', \''.\SQLite3::escapeString($row['file']).'\', '.$row['line'].', \''.\SQLite3::escapeString($row['trace']).'\', \''.
$row['ip'].'\', '.$row['querytime'].', '.$row['time'].', \''. $row['ip'].'\', '.$row['querytime'].', \''.$row['time'].'\', \''.
\SQLite3::escapeString($row['server_name']).'\', '.($row['server_port'] ? $row['server_port'] : 'NULL').', \''.\SQLite3::escapeString($row['request_uri']).'\', \''.\SQLite3::escapeString($row['post']).'\', \''. \SQLite3::escapeString($row['server_name']).'\', '.($row['server_port'] ? $row['server_port'] : 'NULL').', \''.\SQLite3::escapeString($row['request_uri']).'\', \''.\SQLite3::escapeString($row['post']).'\', \''.
\SQLite3::escapeString($row['http_referer']).'\', \''.\SQLite3::escapeString($row['http_user_agent']).'\', '.($row['user'] ? $row['user'] : 'NULL').','.true.');')){ \SQLite3::escapeString($row['http_referer']).'\', \''.\SQLite3::escapeString($row['http_user_agent']).'\', '.($row['user'] ? $row['user'] : 'NULL').','.true.');')){
new \SYSTEM\LOG\ERROR('failed to insert into log archiev'); new \SYSTEM\LOG\ERROR('failed to insert into log archiev');
return cronstatus::CRON_STATUS_FAIL; return cronstatus::CRON_STATUS_FAIL;
} }
if($j > 5000){ //Delete single
$j = 0; if(!\SYSTEM\DBD\SYS_LOG_DEL::QI(array($row['ID']))){
set_time_limit(30);
if(!$con->exec('end transaction')){
new \SYSTEM\LOG\ERROR('failed to insert into log archiev');
return cronstatus::CRON_STATUS_FAIL;};
$con->exec('begin transaction');
}
}
set_time_limit(30);
if(!$con->exec('end transaction')){
new \SYSTEM\LOG\ERROR('failed to insert into log archiev');
return cronstatus::CRON_STATUS_FAIL;};
set_time_limit(30);
//delete from database
if(!\SYSTEM\DBD\SYS_LOG_MONTH_DEL::QI(array($oldest['month'],$oldest['year']))){
new \SYSTEM\LOG\ERROR('failed to delete log entries'); new \SYSTEM\LOG\ERROR('failed to delete log entries');
return cronstatus::CRON_STATUS_FAIL;} return cronstatus::CRON_STATUS_FAIL;}
}
set_time_limit(30);
if(!$con->exec('end transaction')){
new \SYSTEM\LOG\ERROR('failed to insert into log archiev');
return cronstatus::CRON_STATUS_FAIL;};
return cronstatus::CRON_STATUS_SUCCESFULLY; return cronstatus::CRON_STATUS_SUCCESFULLY;
} }

11
dbd/qq/SYS_LOG_DEL.php Normal file
View File

@ -0,0 +1,11 @@
<?php
namespace SYSTEM\DBD;
class SYS_LOG_DEL extends \SYSTEM\DB\QP {
protected static function query(){
return new \SYSTEM\DB\QQuery(get_class(),
//pg
'DELETE FROM '.\SYSTEM\DBD\system_log::NAME_PG.' WHERE "ID" = $1;',
//mys
'DELETE FROM '.\SYSTEM\DBD\system_log::NAME_MYS.' WHERE ID = ?;'
);}}

4
dbd/qq/SYS_LOG_MONTH.php
View File

@ -5,7 +5,7 @@ class SYS_LOG_MONTH extends \SYSTEM\DB\QP {
protected static function query(){ protected static function query(){
return new \SYSTEM\DB\QQuery(get_class(), return new \SYSTEM\DB\QQuery(get_class(),
//pg //pg
'SELECT extract(epoch from now()) as time_pg,* FROM '.\SYSTEM\DBD\system_log::NAME_PG.' WHERE EXTRACT(MONTH FROM time)::INTEGER = $1 AND EXTRACT(YEAR FROM time)::INTEGER = $2 ORDER BY time ASC', 'SELECT extract(epoch from time) as time_pg,* FROM '.\SYSTEM\DBD\system_log::NAME_PG.' WHERE EXTRACT(MONTH FROM time)::INTEGER = $1 AND EXTRACT(YEAR FROM time)::INTEGER = $2 LIMIT 1000;',//ORDER BY time ASC
//mys //mys
'SELECT * FROM '.\SYSTEM\DBD\system_log::NAME_MYS.' WHERE MONTH(time) = ? AND YEAR(time) = ? ORDER BY time ASC' 'SELECT * FROM '.\SYSTEM\DBD\system_log::NAME_MYS.' WHERE MONTH(time) = ? AND YEAR(time) = ? ORDER BY time ASC LIMIT 1000;'
);}} );}}