Commit 5c179522 authored by Mario's avatar Mario
Browse files

bring back externals (slightly different approach), rename zotfeed to outbox...

bring back externals (slightly different approach), rename zotfeed to outbox for consistency wit AP naming. warning: this commit requires update of addons!
parent b35d95da
Pipeline #446685 passed with stage
in 1 minute and 50 seconds
......@@ -36,7 +36,6 @@ class Cron {
// run queue delivery process in the background
Master::Summon(array('Queue'));
Master::Summon(array('Poller'));
/**
......@@ -206,10 +205,9 @@ class Cron {
// pull in some public posts
/* $disable_discover_tab = get_config('system', 'disable_discover_tab') || get_config('system', 'disable_discover_tab') === false;
$disable_discover_tab = get_config('system', 'disable_discover_tab') || get_config('system', 'disable_discover_tab') === false;
if (!$disable_discover_tab)
Master::Summon(array('Externals'));
*/
Master::Summon(['Externals']);
$restart = false;
......
......@@ -3,6 +3,7 @@
namespace Zotlabs\Daemon;
use Zotlabs\Lib\Activity;
use Zotlabs\Lib\Libzot;
use Zotlabs\Lib\ActivityStreams;
use Zotlabs\Lib\ASCollection;
......@@ -31,25 +32,55 @@ class Externals {
$url = $arr['url'];
}
else {
$networks = ['zot6'];
if (plugin_is_installed('pubcrawl')) {
$networks[] = 'activitypub';
}
stringify_array_elms($networks);
$networks_str = implode(',', $networks);
$randfunc = db_getfunc('RAND');
// fixme this query does not deal with directory realms.
$r = q("select site_url, site_pull from site where site_url != '%s'
and site_flags != %d and site_type = %d
and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1",
//$r = q("select site_url, site_pull from site where site_url != '%s'
//and site_flags != %d and site_type = %d
//and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1",
//dbesc(z_root()),
//intval(DIRECTORY_MODE_STANDALONE),
//intval(SITE_TYPE_ZOT),
//dbesc('hubzilla%')
//);
$r = q("SELECT * FROM hubloc
LEFT JOIN abook ON abook_xchan = hubloc_hash
LEFT JOIN site ON site_url = hubloc_url WHERE
hubloc_network IN ( $networks_str ) AND
abook_xchan IS NULL AND
hubloc_url != '%s' AND
hubloc_updated > '%s' AND
hubloc_primary = 1 AND hubloc_deleted = 0 AND
site_dead = 0
ORDER BY $randfunc LIMIT 1",
dbesc(z_root()),
intval(DIRECTORY_MODE_STANDALONE),
intval(SITE_TYPE_ZOT),
dbesc('hubzilla%')
datetime_convert('UTC', 'UTC', 'now - 30 days')
);
if ($r)
$url = $r[0]['site_url'];
$contact = $r[0];
if ($contact) {
$url = $contact['hubloc_id_url'];
}
}
if (!$url) {
continue;
}
$blacklisted = false;
if (!check_siteallowed($url)) {
if (!check_siteallowed($contact['hubloc_url'])) {
logger('blacklisted site: ' . $url);
$blacklisted = true;
}
......@@ -59,123 +90,65 @@ class Externals {
// make sure we can eventually break out if somebody blacklists all known sites
if ($blacklisted) {
if ($attempts > 20)
if ($attempts > 5)
break;
$attempts--;
continue;
}
if ($url) {
$cl = Activity::get_actor_collections($contact['hubloc_hash']);
if(empty($cl)) {
$cl = get_xconfig($contact['hubloc_hash'], 'activitypub', 'collections');
}
$max = intval(get_config('system', 'max_imported_posts', 30));
if (intval($max)) {
logger('externals: fetching outbox');
$feed_url = $url . '/zotfeed';
$obj = new ASCollection($feed_url, $importer, 0, $max);
$messages = $obj->get();
if ($messages) {
foreach ($messages as $message) {
if (is_string($message)) {
$message = Activity::fetch($message, $importer);
}
$AS = new ActivityStreams($message);
if ($AS->is_valid() && is_array($AS->obj)) {
$item = Activity::decode_note($AS);
Activity::store($importer, $importer['xchan_hash'], $AS, $item, true);
$total++;
}
}
}
logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG);
if (is_array($cl) && array_key_exists('outbox', $cl)) {
$url = $cl['outbox'];
}
else {
$url = str_replace('/channel/', '/outbox/', $contact['hubloc_id_url']);
if ($url) {
$url .= '?top=1';
}
}
}
return;
/* $total = 0;
$attempts = 0;
logger('externals: startup', LOGGER_DEBUG);
// pull in some public posts
while ($total == 0 && $attempts < 3) {
$arr = ['url' => ''];
call_hooks('externals_url_select', $arr);
if ($arr['url']) {
$url = $arr['url'];
}
else {
$randfunc = db_getfunc('RAND');
// fixme this query does not deal with directory realms.
$r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d and site_type = %d and site_dead = 0 order by $randfunc limit 1",
dbesc(z_root()),
intval(DIRECTORY_MODE_STANDALONE),
intval(SITE_TYPE_ZOT)
);
if ($r)
$url = $r[0]['site_url'];
}
$blacklisted = false;
if ($url) {
logger('fetching outbox: ' . $url);
if (!check_siteallowed($url)) {
logger('blacklisted site: ' . $url);
$blacklisted = true;
}
$obj = new ASCollection($url, $importer, 0, 10);
$messages = $obj->get();
$attempts++;
if ($messages) {
foreach ($messages as $message) {
if (is_string($message)) {
$message = Activity::fetch($message, $importer);
}
// make sure we can eventually break out if somebody blacklists all known sites
if ($message['type'] !== 'Create') {
continue;
}
if ($blacklisted) {
if ($attempts > 20)
break;
$attempts--;
continue;
}
// make sure we only fetch top level items
if (isset($message['object']['inReplyTo'])) {
continue;
}
if ($url) {
if ($r[0]['site_pull'] > NULL_DATE)
$mindate = urlencode(datetime_convert('', '', $r[0]['site_pull'] . ' - 1 day'));
else {
$days = get_config('externals', 'since_days');
if ($days === false)
$days = 15;
$mindate = urlencode(datetime_convert('', '', 'now - ' . intval($days) . ' days'));
if ($contact['hubloc_network'] === 'zot6') {
Libzot::fetch_conversation($importer, $message['object']['id']);
$total++;
continue;
}
$feedurl = $url . '/zotfeed?f=&mindate=' . $mindate;
logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG);
$x = z_fetch_url($feedurl);
if (($x) && ($x['success'])) {
q("update site set site_pull = '%s' where site_url = '%s'",
dbesc(datetime_convert()),
dbesc($url)
);
$j = json_decode($x['body'], true);
if ($j['success'] && $j['messages']) {
$sys = get_sys_channel();
foreach ($j['messages'] as $message) {
// on these posts, clear any route info.
$message['route'] = '';
process_delivery(['hash' => 'undefined'], get_item_elements($message),
[['hash' => $sys['xchan_hash']]], false, true);
$total++;
}
logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG);
}
$AS = new ActivityStreams($message);
if ($AS->is_valid() && is_array($AS->obj)) {
$item = Activity::decode_note($AS);
Activity::store($importer, $contact['abook_xchan'], $AS, $item);
$total++;
}
}
}*/
}
logger('fetched messages count: ' . $total);
}
}
return;
}
}
......@@ -135,7 +135,7 @@ class Onepoll {
$url = $cl['outbox'];
}
else {
$url = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']);
$url = str_replace('/poco/', '/outbox/', $contact['xchan_connurl']);
}
if ($url) {
......
......@@ -2162,9 +2162,11 @@ class Activity {
if (array_key_exists('published', $act->data)) {
$s['created'] = datetime_convert('UTC', 'UTC', $act->data['published']);
$s['commented'] = $s['created'];
}
elseif (array_key_exists('published', $act->obj)) {
$s['created'] = datetime_convert('UTC', 'UTC', $act->obj['published']);
$s['commented'] = $s['created'];
}
if (array_key_exists('updated', $act->data)) {
$s['edited'] = datetime_convert('UTC', 'UTC', $act->data['updated']);
......
<?php
namespace Zotlabs\Module;
use App;
use Zotlabs\Lib\Activity;
use Zotlabs\Lib\ActivityStreams;
use Zotlabs\Lib\Config;
use Zotlabs\Lib\ThreadListener;
use Zotlabs\Web\Controller;
use Zotlabs\Web\HTTPSig;
class Zotfeed extends Controller {
function init() {
if (ActivityStreams::is_as_request()) {
if (observer_prohibited(true)) {
killme();
}
$channel = channelx_by_nick(argv(1));
if (!$channel) {
killme();
}
if (intval($channel['channel_system'])) {
killme();
}
$sigdata = HTTPSig::verify(($_SERVER['REQUEST_METHOD'] === 'POST') ? file_get_contents('php://input') : EMPTY_STR);
if ($sigdata['portable_id'] && $sigdata['header_valid']) {
$portable_id = $sigdata['portable_id'];
if (!check_channelallowed($portable_id)) {
http_status_exit(403, 'Permission denied');
}
if (!check_siteallowed($sigdata['signer'])) {
http_status_exit(403, 'Permission denied');
}
observer_auth($portable_id);
}
elseif (Config::get('system', 'require_authenticated_fetch', false)) {
http_status_exit(403, 'Permission denied');
}
$observer_hash = get_observer_hash();
$params = [];
function post() {
$params['begin'] = ((x($_REQUEST, 'date_begin')) ? $_REQUEST['date_begin'] : NULL_DATE);
$params['end'] = ((x($_REQUEST, 'date_end')) ? $_REQUEST['date_end'] : '');
$params['type'] = 'json';
$params['pages'] = ((x($_REQUEST, 'pages')) ? intval($_REQUEST['pages']) : 0);
$params['top'] = ((x($_REQUEST, 'top')) ? intval($_REQUEST['top']) : 0);
$params['direction'] = ((x($_REQUEST, 'direction')) ? dbesc($_REQUEST['direction']) : 'desc'); // unimplemented
$params['cat'] = ((x($_REQUEST, 'cat')) ? escape_tags($_REQUEST['cat']) : '');
$params['compat'] = 1;
$total = items_fetch(
[
'total' => true,
'wall' => 1,
'datequery' => $params['end'],
'datequery2' => $params['begin'],
'direction' => dbesc($params['direction']),
'pages' => $params['pages'],
'order' => dbesc('post'),
'top' => $params['top'],
'cat' => $params['cat'],
'compat' => $params['compat']
], $channel, $observer_hash, CLIENT_MODE_NORMAL, App::$module
);
if ($total) {
App::set_pager_total($total);
App::set_pager_itemspage(30);
}
}
if (App::$pager['unset'] && $total > 30) {
$ret = Activity::paged_collection_init($total, App::$query_string);
}
else {
function get() {
$items = items_fetch(
[
'wall' => 1,
'datequery' => $params['end'],
'datequery2' => $params['begin'],
'records' => intval(App::$pager['itemspage']),
'start' => intval(App::$pager['start']),
'direction' => dbesc($params['direction']),
'pages' => $params['pages'],
'order' => dbesc('post'),
'top' => $params['top'],
'cat' => $params['cat'],
'compat' => $params['compat']
], $channel, $observer_hash, CLIENT_MODE_NORMAL, App::$module
);
$outbox = new Outbox();
return $outbox->init();
if ($items && $observer_hash) {
}
// check to see if this observer is a connection. If not, register any items
// belonging to this channel for notification of deletion/expiration
}
$x = q("select abook_id from abook where abook_channel = %d and abook_xchan = '%s'",
intval($channel['channel_id']),
dbesc($observer_hash)
);
if (!$x) {
foreach ($items as $item) {
if (strpos($item['mid'], z_root()) === 0) {
ThreadListener::store($item['mid'], $observer_hash);
}
}
}
}
$ret = Activity::encode_item_collection($items, App::$query_string, 'OrderedCollection', $total);
}
as_return_and_die($ret, $channel);
}
}
}
......@@ -53,7 +53,7 @@ require_once('include/bbcode.php');
require_once('include/items.php');
define ( 'PLATFORM_NAME', 'hubzilla' );
define ( 'STD_VERSION', '6.3.3' );
define ( 'STD_VERSION', '6.3.4' );
define ( 'ZOT_REVISION', '6.0' );
define ( 'DB_UPDATE_VERSION', 1248 );
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment