Added a simple submit-forwarding feature through the syncing cronjob.
This commit is contained in:
parent
5c477f9b7f
commit
0026b08a33
3 changed files with 163 additions and 0 deletions
13
dfrndir.sql
13
dfrndir.sql
|
@ -199,3 +199,16 @@ CREATE TABLE IF NOT EXISTS `site-scrape` (
|
||||||
KEY `dt_performed` (`dt_performed`)
|
KEY `dt_performed` (`dt_performed`)
|
||||||
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
|
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS `sync-targets` (
|
||||||
|
`base_url` varchar(255) NOT NULL,
|
||||||
|
`pull` bit(1) NOT NULL DEFAULT b'0',
|
||||||
|
`push` bit(1) NOT NULL DEFAULT b'1',
|
||||||
|
PRIMARY KEY (`base_url`),
|
||||||
|
KEY `push` (`push`),
|
||||||
|
KEY `pull` (`pull`)
|
||||||
|
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS `sync-queue` (
|
||||||
|
`url` varchar(255) NOT NULL,
|
||||||
|
PRIMARY KEY (`url`)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ;
|
||||||
|
|
141
include/syncing.php
Normal file
141
include/syncing.php
Normal file
|
@ -0,0 +1,141 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
// Debug stuff.
|
||||||
|
ini_set('display_errors', 1);
|
||||||
|
ini_set('log_errors','0');
|
||||||
|
error_reporting(E_ALL^E_NOTICE);
|
||||||
|
|
||||||
|
$start_syncing = time();
|
||||||
|
|
||||||
|
//Startup.
|
||||||
|
require_once('boot.php');
|
||||||
|
$a = new App;
|
||||||
|
|
||||||
|
//Create a simple log function for CLI use.
|
||||||
|
$verbose = $argv[1] === 'verbose';
|
||||||
|
$msg = function($message, $fatal=false)use($verbose){
|
||||||
|
if($verbose || $fatal) echo($message.PHP_EOL);
|
||||||
|
logger($message);
|
||||||
|
if($fatal) exit(1);
|
||||||
|
};
|
||||||
|
|
||||||
|
//Config.
|
||||||
|
require_once(".htconfig.php");
|
||||||
|
|
||||||
|
//No pushing? Leave... because we haven't implemented pulling yet.
|
||||||
|
if(!$a->config['syncing']['enable_pushing']){
|
||||||
|
$msg('No push support enabled in your settings.', true);
|
||||||
|
}
|
||||||
|
|
||||||
|
//Connect the DB.
|
||||||
|
require_once("dba.php");
|
||||||
|
$db = new dba($db_host, $db_user, $db_pass, $db_data, $install);
|
||||||
|
|
||||||
|
//Find our targets.
|
||||||
|
$targets = q("SELECT * FROM `sync-targets` WHERE `push`=b'1'");
|
||||||
|
if(!count($targets)) $msg('No targets.', true); //No targets, means no work.
|
||||||
|
|
||||||
|
//Get our batch of URL's.
|
||||||
|
$batch = q("SELECT * FROM `sync-queue` LIMIT %u", intval($a->config['syncing']['max_push_items']));
|
||||||
|
if(!count($batch)) $msg('Empty queue.', true); //No batch, means no work.
|
||||||
|
|
||||||
|
//Close the connection for now. Process forking and DB connections are not the best of friends.
|
||||||
|
$db->getdb()->close();
|
||||||
|
|
||||||
|
//Create a thread for each target we want to serve push messages to.
|
||||||
|
//No good creating more, because it would stress their server too much.
|
||||||
|
$threadc = count($targets);
|
||||||
|
$threads = array();
|
||||||
|
|
||||||
|
//Do we only have 1 target? No need for threads.
|
||||||
|
if($threadc === 1){
|
||||||
|
//Pretend to be worker #1.
|
||||||
|
$pid = 0;
|
||||||
|
$i = 0;
|
||||||
|
$main = true;
|
||||||
|
$msg('No threads needed. Only one pushing target.');
|
||||||
|
}
|
||||||
|
|
||||||
|
//When we need threads.
|
||||||
|
else{
|
||||||
|
|
||||||
|
//POSIX threads only.
|
||||||
|
if(!function_exists('pcntl_fork')){
|
||||||
|
$msg('Error: no pcntl_fork support. Are you running a different OS? Report an issue please.', true);
|
||||||
|
}
|
||||||
|
|
||||||
|
//Debug...
|
||||||
|
$items = count($batch);
|
||||||
|
$msg("Creating $threadc push threads for $items items.");
|
||||||
|
|
||||||
|
//Loop while we need more threads.
|
||||||
|
for($i = 0; $i < $threadc; $i++){
|
||||||
|
|
||||||
|
$pid = pcntl_fork();
|
||||||
|
if($pid === -1) $msg('Error: something went wrong with the fork. '.pcntl_strerror(), true);
|
||||||
|
|
||||||
|
//You're a child, go do some labor!
|
||||||
|
if($pid === 0) break;
|
||||||
|
|
||||||
|
//Store the list of PID's.
|
||||||
|
if($pid > 0) $threads[] = $pid;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
//Are we the main thread?
|
||||||
|
$main = $pid !== 0;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
//The work for child processes.
|
||||||
|
if($pid === 0){
|
||||||
|
|
||||||
|
//Lets be nice, we're only doing a background job here...
|
||||||
|
pcntl_setpriority(5);
|
||||||
|
|
||||||
|
//Find our target's submit URL.
|
||||||
|
$submit = $targets[$i]['base_url'].'/submit';
|
||||||
|
|
||||||
|
foreach($batch as $item){
|
||||||
|
set_time_limit(30); //This should work for 1 submit.
|
||||||
|
$msg("Submitting {$item['url']} to $submit");
|
||||||
|
fetch_url($submit.'?url='.bin2hex($item['url']));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
//The main process.
|
||||||
|
if($main){
|
||||||
|
|
||||||
|
//Wait for all child processes.
|
||||||
|
$all_good = true;
|
||||||
|
foreach($threads as $pid){
|
||||||
|
pcntl_waitpid($pid, $status);
|
||||||
|
if($status !== 0){
|
||||||
|
$all_good = false;
|
||||||
|
$msg("Bad process return value $pid:$status");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//If we did not have any "threading" problems.
|
||||||
|
if($all_good){
|
||||||
|
|
||||||
|
//Reconnect
|
||||||
|
$db = new dba($db_host, $db_user, $db_pass, $db_data, $install);
|
||||||
|
|
||||||
|
//Create a query for deleting this queue.
|
||||||
|
$where = array();
|
||||||
|
foreach($batch as $item) $where[] = dbesc($item['url']);
|
||||||
|
$where = "WHERE `url` IN ('".implode("', '", $where)."')";
|
||||||
|
|
||||||
|
//Remove the items from queue.
|
||||||
|
q("DELETE FROM `sync-queue` $where LIMIT %u", intval($a->config['syncing']['max_push_items']));
|
||||||
|
$msg('Removed items from queue.');
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
//Log the time it took.
|
||||||
|
$time = time() - $start_syncing;
|
||||||
|
$msg("Syncing completed. Took $time seconds.");
|
||||||
|
|
||||||
|
}
|
|
@ -4,7 +4,16 @@ require_once('include/submit.php');
|
||||||
|
|
||||||
function submit_content(&$a) {
|
function submit_content(&$a) {
|
||||||
|
|
||||||
|
//Decode the URL.
|
||||||
$url = hex2bin(notags(trim($_GET['url'])));
|
$url = hex2bin(notags(trim($_GET['url'])));
|
||||||
|
|
||||||
|
//Currently we simply push RAW URL's to our targets.
|
||||||
|
//If we support it that is.
|
||||||
|
if($a->config['syncing']['enable_pushing']){
|
||||||
|
q("INSERT INTO `sync-queue` (`url`) VALUES ('%s')", dbesc($url));
|
||||||
|
}
|
||||||
|
|
||||||
|
//Run the submit sequence.
|
||||||
run_submit($url);
|
run_submit($url);
|
||||||
exit;
|
exit;
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue