Commit e608b727 authored by Luc Didry's avatar Luc Didry

Use API to fetch toots

+ video handling
+ attachments cache

This commit is dedicated to Éloïs, who is supporting me in Ğ1
Many thanks :-)
parent 935d1b9c
Pipeline #23537 passed with stages
in 18 seconds
Revision history for Last
0.2.0 2017-09-24
- Use API to fetch toots
- Video attachments handling
- Add attachments cache
0.1.2 2017-09-15
- Support images on different hosts than the Mastodon pod
......
......@@ -7,7 +7,7 @@ use Mojo::File;
use Mojo::URL;
use Mojo::UserAgent;
use Mojo::Util qw(decode encode url_unescape);
use Mojo::JSON qw(decode_json);
use Mojo::JSON qw(decode_json encode_json);
use Mojolicious;
use Archive::Zip qw( :ERROR_CODES :CONSTANTS );
use DateTime;
......@@ -20,7 +20,6 @@ use FindBin qw($Bin);
use Text::Slugify 'slugify';
use XML::Atom::SimpleFeed;
use POSIX;
use Version::Compare;
# Get config
my $m = Mojolicious->new();
......@@ -163,40 +162,26 @@ $urls->each(
## Get only the pod's hostname and scheme
my $host = Mojo::URL->new($i)->path('');
## Get pod version
my $pod = $host;
unless (defined $versions{$pod}) {
$versions{$pod} = $ua->get($pod->path('/api/v1/instance'))->result->json->{version};
}
my $cache = (-e "cache/$id.html");
my $cache = (-e "cache/$id.json");
my $msg = "Processing $i";
$msg .= " from cache" if $cache;
say $msg;
my $res;
$res = $ua->get($i)->result unless $cache;
$res = $ua->get($host->path("/api/v1/statuses/$id"))->result unless $cache;
if ($cache || $res->is_success) {
Mojo::File->new("cache/$id.html")->spurt(encode 'UTF-8', $res->dom->to_string) unless $cache;
Mojo::File->new("cache/$id.json")->spurt(encode_json($res->json)) unless $cache;
## Get only the targeted toot, not the replies
my $dom = ($cache) ? Mojo::DOM->new(decode('UTF-8', Mojo::File->new("cache/$id.html")->slurp)) : $res->dom;
$dom = $dom->find('.entry-center .detailed-status')->first;
my $json = ($cache) ? decode_json(Mojo::File->new("cache/$id.json")->slurp) : $res->json;
## Get the content of the toot
my $content = $dom->find('.e-content')->first;
my $content = Mojo::DOM->new($json->{content});
## Get the metadata (ie date and time) of the toot
my $dt = $f->parse_datetime($dom->find('.dt-published')->first->val)->set_locale($config->{language});
my $dt = $f->parse_datetime($json->{created_at})->set_locale($config->{language});
## Get the attachments of the toot
my $attach;
if (&Version::Compare::version_compare($versions{$pod}, '1.6.0') == 1) {
my $gal = $dom->find('div[data-component="MediaGallery"]');
$attach = ($gal->size) ? c(@{decode_json($gal->first->attr('data-props'))->{media}}) : c();
} else {
$attach = $dom->find('a.u-photo');
}
my $attach = c(@{$json->{media_attachments}});
## Remove style attribute
delete $content->attr->{'style'};
## Replace emoji with unicode characters
$content->find('img.emojione')->each(
sub {
......@@ -204,7 +189,7 @@ $urls->each(
$e->replace($e->attr('alt'));
}
);
## Remove the MercrediFiction hashtag
## Remove the configured hashtags
$content->find('a.mention.hashtag')->each(
sub {
my ($e, $num) = @_;
......@@ -224,37 +209,55 @@ $urls->each(
sub {
my ($e, $inum) = @_;
my $src;
if (&Version::Compare::version_compare($versions{$pod}, '1.6.0') == 1) {
$src = $e->{url};
} else {
$src = $e->attr('href');
}
my $src = $e->{url};
$src =~ s#\?(.*)##;
my $q = $1;
$src =~ m#/([^/]*)$#;
my $n = $1;
$src = Mojo::URL->new($src);
my $obj = (defined($src->host)) ? $src : $host->path($src)->query($q);
say sprintf(" Fetching image: %s", $obj->to_string);
my $r = $ua->get($obj->to_string)->result;
if ($r->is_success) {
my $body = $r->body;
my $rmime = $r->headers->content_type;
my $mime = $rmime;
$mime =~ s#.*/##;
my $img = "img/$n.$mime";
Mojo::File->new("public/$img")->spurt($body);
push @imgs, "<img class=\"u-max-full-width\" src=\"$img\" alt=\"\">";
Mojo::File->new("public/epub/OPS/$img")->spurt($body);
## Attachments cache
my $acache = (-e "cache/img/$n" && -e "cache/img/$n.meta");
my $msg = sprintf(" Fetching image: %s", $src);
$msg .= " from cache" if $acache;
say $msg;
my $img = "img/$n";;
my $rmime;
if ($acache) {
# Get file metadata from cache
$rmime = decode_json(Mojo::File->new("cache/$img.meta")->slurp);
# Copy file
Mojo::File->new("cache/$img")->copy_to("public/$img")->copy_to("public/epub/OPS/$img");
} else {
my $r = $ua->get($src)->result;
if ($r->is_success) {
my $body = $r->body;
$rmime = $r->headers->content_type;
# Create cache
mkdir 'cache/img' unless -d 'cache/img';
Mojo::File->new("cache/$img")->spurt($body);
Mojo::File->new("cache/$img.meta")->spurt(encode_json($rmime));
# Copy file
Mojo::File->new("public/$img")->spurt($body);
Mojo::File->new("public/epub/OPS/$img")->spurt($body);
} elsif ($r->is_error) {
die sprintf("Error while fetching %s: %s", $src, $m->dumper($r->message));
}
}
if ($e->{type} eq 'image') {
push @imgs, "<img class=\"u-max-full-width\" src=\"$img\" alt=\"\">";
} elsif ($e->{type} eq 'video') {
push @imgs, "<video class=\"u-max-full-width\" src=\"$img\" alt=\"\"></video>";
}
if ($e->{type} eq 'image') {
push @imgs2, "<img class=\"u-max-full-width\" src=\"../$img\" alt=\"\" />";
$manifest->append_content(" <item id=\"i$num-$inum\" href=\"$img\" media-type=\"$rmime\" />\n");
$n++
} elsif ($r->is_error) {
die sprintf("Error while fetching %s: %s", $obj->to_string, $m->dumper($r->message));
} elsif ($e->{type} eq 'video') {
push @imgs2, "<video class=\"u-max-full-width\" src=\"../$img\" alt=\"\" ></video>";
}
$manifest->append_content(" <item id=\"i$num-$inum\" href=\"$img\" media-type=\"$rmime\" />\n");
}
);
my $attachments = (scalar @imgs) ? "<div><p>@imgs</p></div>" : '';
......
......@@ -7,4 +7,3 @@ requires 'File::Copy::Recursive';
requires 'XML::Atom::SimpleFeed';
requires 'Archive::Zip';
requires 'Text::Slugify';
requires 'Version::Compare';
......@@ -1101,15 +1101,6 @@ DISTRIBUTIONS
base 0
lib 0
perl 5.008
Version-Compare-0.14
pathname: T/TE/TEX/Version-Compare-0.14.tar.gz
provides:
Version::Compare 0.14
requirements:
ExtUtils::MakeMaker 6.30
Test::More 0
strict 0
warnings 0
XML-Atom-SimpleFeed-0.900
pathname: A/AR/ARISTOTLE/XML-Atom-SimpleFeed-0.900.tar.gz
provides:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment