From 40202ea948363266d73c4f16b52d3a8d2a3aad09 Mon Sep 17 00:00:00 2001 From: Beanow Date: Tue, 1 Jul 2014 03:27:56 +0200 Subject: [PATCH] Made a first version of import functionality. * A very crude extraction step that creates a file with URLs. * A processor that reduces the file to nothing as it imports them. --- include/submit.php | 195 +++++++++++++++++++++++++++++++++++++++++++++ mod/import.php | 135 +++++++++++++++++++++++++++++++ mod/submit.php | 167 +------------------------------------- view/import.tpl | 14 ++++ 4 files changed, 347 insertions(+), 164 deletions(-) create mode 100644 include/submit.php create mode 100644 mod/import.php create mode 100644 view/import.tpl diff --git a/include/submit.php b/include/submit.php new file mode 100644 index 00000000..129b483d --- /dev/null +++ b/include/submit.php @@ -0,0 +1,195 @@ + 1) { + q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1", + intval($r[0]['id']) + ); + q("DELETE FROM `profile` WHERE `id` = %d LIMIT 1", + intval($r[0]['id']) + ); + } + + } + + if($parms['tags']) { + $arr = explode(' ', $parms['tags']); + if(count($arr)) { + foreach($arr as $t) { + $t = strip_tags(trim($t)); + $t = substr($t,0,254); + + if(strlen($t)) { + $r = q("SELECT `id` FROM `tag` WHERE `term` = '%s' and `nurl` = '%s' LIMIT 1", + dbesc($t), + dbesc($nurl) + ); + if(! count($r)) { + $r = q("INSERT INTO `tag` (`term`, `nurl`) VALUES ('%s', '%s') ", + dbesc($t), + dbesc($nurl) + ); + } + } + } + } + } + + require_once("Photo.php"); + + $photo_failure = false; + + $img_str = fetch_url($photo,true); + $img = new Photo($img_str); + if($img) { + $img->scaleImageSquare(80); + $r = $img->store($profile_id); + } + if($profile_id) { + $r = q("UPDATE `profile` SET `photo` = '%s' WHERE `id` = %d LIMIT 1", + dbesc($a->get_baseurl() . '/photo/' . $profile_id . '.jpg'), + intval($profile_id) + ); + } + else{ + nuke_record($url); + return false; + } + return true; + +} + + +function nuke_record($url) { + + $nurl = str_replace(array('https:','//www.'), array('http:','//'), $url); + + $r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' OR `nurl` = '%s' ) ", + dbesc($url), + dbesc($nurl) + ); + + if(count($r)) { + foreach($r as $rr) { + q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1", + intval($rr['id']) + ); + q("DELETE FROM `profile` WHERE `id` = %d LIMIT 1", + intval($rr['id']) + ); + } + } + return; +} \ No newline at end of file diff --git a/mod/import.php b/mod/import.php new file mode 100644 index 00000000..63451c82 --- /dev/null +++ b/mod/import.php @@ -0,0 +1,135 @@ + is_file('.htimport') ? ' (present)' : '' + )); + +} + +function import_post(&$a) +{ + + //Get our input. + $url = $_POST['url']; + $page = intval($_POST['page']); + $batch = $_POST['batch_submit']; + + //Directory + $file = realpath(__DIR__.'/..').'/.htimport'; + + //Per batch setting. + $perPage = 200; + $perBatch = 10; + + if($batch){ + + require_once('include/submit.php'); + + //First get all data from file. + $data = file_get_contents($file); + $list = explode("\r\n", $data); + + //Fresh batch? + if(!isset($_SESSION['import_progress'])){ + + $_SESSION['import_progress'] = true; + $_SESSION['import_success'] = 0; + $_SESSION['import_failed'] = 0; + $_SESSION['import_total'] = 0; + notice("Started new batch. "); + + } + + //Make sure we can use try catch for all sorts of errors. + set_error_handler(function($errno, $errstr='', $errfile='', $errline='', $context=array()){ + if((error_reporting() & $errno) == 0){ return; } + throw new \Exception($errstr, $errno); + }); + + for($i=0; $i<$perBatch; $i++){ + if($url = array_shift($list)){ + set_time_limit(20); + $_SESSION['import_total']++; + $_SESSION['import_failed']++; + try{ + if(run_submit($a, $url)){ + $_SESSION['import_failed']--; + $_SESSION['import_success']++; + } + }catch(\Exception $ex){/* We tried... */} + } + else break; + } + + $left = count($list); + + if($left > 0){ + notice("$left items left in batch."); + file_put_contents($file, implode("\r\n", $list)); + $fid = uniqid('autosubmit_'); + echo '
'. + ''; + } else { + $s = $_SESSION['import_success']; + $total = $_SESSION['import_total']; + $errors = $_SESSION['import_failed']; + notice("Completed batch! $s / $total success. $errors errors."); + unlink($file); + unset($_SESSION['import_progress']); + } + + return; + + } + elseif($url && $page){ + + $result = fetch_url($url."/lsearch?p=$page&n=$perPage&search=.*"); + if($result) + $data = json_decode($result); + else + $data = false; + + if($data){ + + $rows = ''; + foreach($data->results as $profile){ + $rows .= $profile->url."\r\n"; + } + file_put_contents($file, $rows, $page > 0 ? FILE_APPEND : 0); + + $progress = min((($page+1) * $perPage), $data->total); + notice("Imported ".$progress."/".$data->total." URLs."); + + if($progress !== $data->total){ + $fid = uniqid('autosubmit_'); + echo + '
'. + ''. + ''. + // ''. + '
'. + ''; + + } else { + goaway($a->get_baseurl().'/import'); + } + + } + + } + +} \ No newline at end of file diff --git a/mod/submit.php b/mod/submit.php index 64f2596b..f15c1058 100755 --- a/mod/submit.php +++ b/mod/submit.php @@ -1,172 +1,11 @@ 1) { - q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1", - intval($r[0]['id']) - ); - q("DELETE FROM `profile` WHERE `id` = %d LIMIT 1", - intval($r[0]['id']) - ); - } - - } - - if($parms['tags']) { - $arr = explode(' ', $parms['tags']); - if(count($arr)) { - foreach($arr as $t) { - $t = strip_tags(trim($t)); - $t = substr($t,0,254); - - if(strlen($t)) { - $r = q("SELECT `id` FROM `tag` WHERE `term` = '%s' and `nurl` = '%s' LIMIT 1", - dbesc($t), - dbesc($nurl) - ); - if(! count($r)) { - $r = q("INSERT INTO `tag` (`term`, `nurl`) VALUES ('%s', '%s') ", - dbesc($t), - dbesc($nurl) - ); - } - } - } - } - } - - require_once("Photo.php"); - - $photo_failure = false; - - $img_str = fetch_url($photo,true); - $img = new Photo($img_str); - if($img) { - $img->scaleImageSquare(80); - $r = $img->store($profile_id); - } - if($profile_id) { - $r = q("UPDATE `profile` SET `photo` = '%s' WHERE `id` = %d LIMIT 1", - dbesc($a->get_baseurl() . '/photo/' . $profile_id . '.jpg'), - intval($profile_id) - ); - } - else - nuke_record($url); + run_submit($a, $url); exit; } diff --git a/view/import.tpl b/view/import.tpl new file mode 100644 index 00000000..fe082712 --- /dev/null +++ b/view/import.tpl @@ -0,0 +1,14 @@ +
+

Mirror a directory

+
+ + + + +
+
+
+ + +
+
\ No newline at end of file