Commit a978f044 authored by Luc Didry's avatar Luc Didry

Add "merge toots" and "customize toots' content" abilities

This commit is dedicated to Liandri, who is supporting me with food.
Many thanks :-)
parent 57f19533
Pipeline #41685 passed with stages
in 1 minute
......@@ -14,6 +14,7 @@ pages:
- cp last.conf last/
- if [[ -d local ]]; then mv local last; fi
- if [[ -d cache ]]; then mv cache last; fi
- if [[ -e LastCustom.pm ]]; then cp LastCustom.pm last; fi
- cd last
- carton install
- carton exec ./build.pl
......
Revision history for Last
0.6.0 2018-03-07
- Allow to merge multiple toots
- Allow to modify the content of the toot through a custom module
0.5.0 2017-12-31
- Change CI to use the latest upstream version of Last
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package LastCustom;
use Mojo::Base -strict;
sub custom_filter {
my $content = shift;
my $id = shift;
# Exemple of modification:
# $content =~ s/Onoyo/Anoyo/g if $id == 2583066;
return $content;
}
1;
......@@ -14,12 +14,18 @@ The [logo](https://openclipart.org/detail/267534/viking-hat) comes from [Carolem
Fork this project, modify `last.conf`, then commit and push. Enjoy.
To add new toots, add them in `urls` and then commit and push.
To add new toots, add them in `urls` and then commit and push. You can merge multiple toots by adding them like this:
```
[ 'https://framapiaf.org/@framasky/99445908349445234', 'https://framapiaf.org/@framasky/99445923365976338' ],
```
If you want to change the aspect of the generated web page and epub, copy the default theme to a new directory in `themes/`, modify it and choose this new theme in the configuration file.
If you want to handle [Markdown syntax](https://daringfireball.net/projects/markdown/syntax) in your toots, you'll need to install pandoc.
If you want to modify the content of the toots, modify `LastCustom.pm` file.
## Example
Please, go to <https://luc.frama.io/last/> to see how it looks.
......
......@@ -20,7 +20,20 @@ use FindBin qw($Bin);
use Text::Slugify 'slugify';
use XML::Atom::SimpleFeed;
use Pandoc;
use Scalar::Util qw(reftype);
use POSIX;
use lib '.';
use vars qw($lc_loaded);
BEGIN {
eval "use LastCustom";
if ($@) {
$lc_loaded = 0;
} else {
$lc_loaded = 1;
}
print $lc_loaded, "\n";
}
# Get config
my $m = Mojolicious->new();
......@@ -47,6 +60,7 @@ my $license = $config->{license};
my $index = open_index();
## Atom feed
$url = Mojo::URL->new($config->{public_url});
my $iso = DateTime::Format::RFC3339->format_datetime(DateTime->now());
my $feed = XML::Atom::SimpleFeed->new(
title => $config->{title},
......@@ -149,189 +163,18 @@ my $urls = c(@{$config->{urls}});
mkdir 'public/epub/OPS/text';
mkdir 'cache' unless -e 'cache';
# URLs processing
## URLs processing
# fetching toots, attachments, formatting, creating html, atom feed, epub…
my $n = 0;
my %versions = ();
$urls->each(
sub {
my ($i, $num) = @_;
## Get the ID of the toot
my $id = $i;
$id =~ s#^.*/([^/]+)$#$1#;
## Get only the pod's hostname and scheme
my $host = Mojo::URL->new($i)->path('');
my $cache = (-e "cache/$id.json");
my $msg = "Processing $i";
$msg .= " from cache" if $cache;
say $msg;
my $res;
$res = $ua->get($host->path("/api/v1/statuses/$id"))->result unless $cache;
if ($cache || $res->is_success) {
Mojo::File->new("cache/$id.json")->spurt(encode_json($res->json)) unless $cache;
## Get only the targeted toot, not the replies
my $json = ($cache) ? decode_json(Mojo::File->new("cache/$id.json")->slurp) : $res->json;
## Get the content of the toot
my $content = Mojo::DOM->new($json->{content});
## Get the metadata (ie date and time) of the toot
my $dt = $f->parse_datetime($json->{created_at})->set_locale($config->{language});
$dt->set_time_zone($config->{timezone}) if defined $config->{timezone};
## Get the attachments of the toot
my $attach = c(@{$json->{media_attachments}});
## Remove style attribute
## Replace emoji with unicode characters
$content->find('img.emojione')->each(
sub {
my ($e, $num) = @_;
$e->replace($e->attr('alt'));
}
);
## Remove the configured hashtags
$content->find('a.mention.hashtag')->each(
sub {
my ($e, $num) = @_;
my $href = $e->attr('href');
$e->remove if $href =~ m#tags/($regex)$#;
}
) if $regex;
## Format date and time
my $date = $dt->format_cldr($l->date_format_full);
my $time = $dt->format_cldr($l->time_format_medium);
## Store the attached images locally
my @imgs;
my @imgs2;
$attach->each(
sub {
my ($e, $inum) = @_;
my $src = $e->{url};
$src =~ s#\?(.*)##;
my $q = $1;
$src =~ m#/([^/]*)$#;
my $n = $1;
$src = Mojo::URL->new($src);
## Attachments cache
my $acache = (-e "cache/img/$n" && -e "cache/img/$n.meta");
my $msg = sprintf(" Fetching image: %s", $src);
$msg .= " from cache" if $acache;
say $msg;
my $img = "img/$n";;
my $rmime;
if ($acache) {
# Get file metadata from cache
$rmime = decode_json(Mojo::File->new("cache/$img.meta")->slurp);
# Copy file
Mojo::File->new("cache/$img")->copy_to("public/$img")->copy_to("public/epub/OPS/$img");
} else {
my $r = $ua->get($src)->result;
if ($r->is_success) {
my $body = $r->body;
$rmime = $r->headers->content_type;
# Create cache
mkdir 'cache/img' unless -d 'cache/img';
Mojo::File->new("cache/$img")->spurt($body);
Mojo::File->new("cache/$img.meta")->spurt(encode_json($rmime));
# Copy file
Mojo::File->new("public/$img")->spurt($body);
Mojo::File->new("public/epub/OPS/$img")->spurt($body);
} elsif ($r->is_error) {
die sprintf("Error while fetching %s: %s", $src, $m->dumper($r->message));
}
}
if ($e->{type} eq 'image') {
push @imgs, "<img class=\"u-max-full-width\" src=\"$img\" alt=\"\">";
} elsif ($e->{type} eq 'video') {
push @imgs, "<video class=\"u-max-full-width\" src=\"$img\" alt=\"\"></video>";
}
if ($e->{type} eq 'image') {
push @imgs2, "<img class=\"u-max-full-width\" src=\"../$img\" alt=\"\" />";
} elsif ($e->{type} eq 'video') {
push @imgs2, "<video class=\"u-max-full-width\" src=\"../$img\" alt=\"\" ></video>";
}
$manifest->append_content(" <item id=\"i$num-$inum\" href=\"$img\" media-type=\"$rmime\" />\n");
}
);
my $attachments = (scalar @imgs) ? "<div><p>@imgs</p></div>" : '';
my $attachments2 = (scalar @imgs2) ? "<div><p>@imgs2</p></div>" : '';
## Mix date, time, content and attachments
if ($config->{markdown} && $content =~ m#([^%])?%md#) {
$content =~ s#%%md#%%LAST_MARKDOWN%%#g;
$content =~ s#%md##g;
$content =~ s#%%LAST_MARKDOWN%%#%md#g;
# Images
$content =~ s#!\[(.*?)\]\(<a href="([^"]*)".*?</a>\)#![$1]($2)#gm;
# Links with text
$content =~ s#\[(.*?)\]\(<a href="([^"]*)".*?</a>\)#[$1]($2)#gm;
# Links
$content =~ s#&lt;<a href="([^"]*)".*?</a>&gt;#<$1>#gm;
my $md;
my $html = encode('UTF-8', $content);
pandoc -f => 'markdown', -t => 'html', { in => \$html, out => \$md };
say $md;
$content = decode('UTF-8', $md);
}
$content =~ s#<p></p>|</br>##g;
$content =~ s#<br>#<br />#g;
my $append = "\n <article><a href=\"#$id\"><h3 id=\"$id\">$date <span style=\"font-size: 0.5em;\">$time</span></h3></a>$content$attachments<div><a href=\"$i\"><em>Source</em></a></div></article><hr>\n";
## Insert the toot
$c->append_content($append);
## Paginate
if ($pagination && (($num % $pagination) == 0 || ($num == $urls->size))) {
my $prec = ($page == 2) ? 'index' : 'page'.($page - 1);
if ($num == $urls->size) {
$c->append_content("\n <p class=\"u-pull-left\"><a class=\"button button-primary\" href=\"$prec.html\">⇐ Page précédente</a></p>");
} else {
$page++;
if ($page == 2) {
$c->append_content("\n <p class=\"u-pull-right\"><a class=\"button button-primary\" href=\"page$page.html\">Page suivante ⇒</a></p>");
} else {
$c->append_content("\n <p class=\"u-pull-left\"><a class=\"button button-primary\" href=\"$prec.html\">⇐ Page précédente</a></p><p class=\"u-pull-right\"><a class=\"button button-primary\" href=\"page$page.html\">Page suivante ⇒</a></p>");
}
$c->append_content(" ");
$file->spurt(encode('UTF-8', $index));
$file = Mojo::File->new("public/page$page.html");
$index = open_index();
$c = $index->find('#content')->first;
$c->append_content(" <p><a href=\"".slugify($config->{title}).".epub\">Epub</a></p>\n");
}
}
## Create entries for the atom feed
push @entries, {
title => "$date $time",
id => $id,
updated => DateTime::Format::RFC3339->format_datetime($dt),
content => $append
};
## Epub
push @pages, {
num => $num,
date => $date,
time => $time,
content => "<h1>$date <span style=\"font-size: 0.5em;\">$time</span></h1><hr/>$content$attachments2<div><a href=\"$i\"><em>Source</em></a></div>"
};
} elsif ($res->is_error) {
die "Error while fetching $i: $res->message";
}
my ($e, $nb) = @_;
my ($content, $i, $date, $time, $attachments, $id, $url, $dt, $num, $attachments2) = process_entry($e, $nb);
format_and_insert_toot($content, $i, $date, $time, $attachments, $id, $url, $dt, $num, $attachments2);
}
);
$c->append_content(" ");
# Write the index.html file
......@@ -383,9 +226,9 @@ $pa->each(
);
my $num = $pa->size + 1;
$manifest->append_content(" ");
$spine->append_content(" <itemref idref=\"nav\" />\n ");
$navmap->append_content(" <navPoint id=\"NavPoint-$num\" playOrder=\"$num\"><navLabel><text>Table des matières</text></navLabel><content src=\"nav.xhtml\"/></navPoint>\n");
$ol->append_content("\n <li><a href=\"nav.xhtml\">Table des matières</a></li>\n ");
$spine->append_content(" <itemref idref=\"nav\" />\n ");
$navmap->append_content(" <navPoint id=\"NavPoint-$num\" playOrder=\"$num\"><navLabel><text>Table des matières</text></navLabel><content src=\"nav.xhtml\"/></navPoint>\n");
$ol->append_content("\n <li><a href=\"nav.xhtml\">Table des matières</a></li>\n ");
Mojo::File->new('public/epub/OPS/toc.ncx')->spurt(encode('UTF-8', $toc->to_string));
Mojo::File->new('public/epub/OPS/nav.xhtml')->spurt(encode('UTF-8', $nav->to_string));
Mojo::File->new('public/epub/OPS/content.opf')->spurt(encode('UTF-8', $opf->to_string));
......@@ -441,3 +284,270 @@ sub open_index {
return $index;
}
sub process_entry {
my ($i, $num) = @_;
if (reftype(\$i) eq 'SCALAR') {
## Get the ID of the toot
my $id = $i;
$id =~ s#^.*/([^/]+)$#$1#;
## Get only the pod's hostname and scheme
my $host = Mojo::URL->new($i)->path('');
my $cache = (-e "cache/$id.json");
my $msg = "Processing $i";
$msg .= " from cache" if $cache;
say $msg;
my $res;
$res = $ua->get($host->path("/api/v1/statuses/$id"))->result unless $cache;
if ($cache || $res->is_success) {
Mojo::File->new("cache/$id.json")->spurt(encode_json($res->json)) unless $cache;
## Get only the targeted toot, not the replies
my $json = ($cache) ? decode_json(Mojo::File->new("cache/$id.json")->slurp) : $res->json;
my ($content, $dt, $attach) = extract_from_json($json);
$content = filter_content($content);
my ($date, $time) = format_time($dt);
my ($attachments, $attachments2) = fetch_attachments($attach, $num);
## Mix date, time, content and attachments
# Modifs persos
if ($lc_loaded) {
$content = LastCustom::custom_filter($content, $id);
}
if ($config->{markdown} && $content =~ m#([^%])?%md#) {
$content = process_md($content);
}
return ($content, $i, $date, $time, $attachments, $id, $url, $dt, $num, $attachments2);
} elsif ($res->is_error) {
die "Error while fetching $i: $res->message";
}
} else {
my ($content, $attachments, $attachments2, $date, $time, $id, $url, $dt) = ('', '', '');
c(@{$i})->each(
sub {
my ($e, $nb) = @_;
my ($tmp_content, $tmp_i, $tmp_date, $tmp_time, $tmp_attachments, $tmp_id, $tmp_url, $tmp_dt, $tmp_num, $tmp_attachments2) = process_entry($e, $nb);
if ($nb == 1) {
$i = $tmp_i;
$date = $tmp_date;
$time = $tmp_time;
$id = $tmp_id;
$url = $tmp_url;
$dt = $tmp_dt;
$content = $tmp_content;
$attachments = $tmp_attachments;
$attachments2 = $tmp_attachments2;
} else {
if ($tmp_content) {
my $tmp_dom = Mojo::DOM->new($tmp_content)->at('p')->content;
my $dom = Mojo::DOM->new($content)->at('p')->content;
$content = Mojo::DOM->new($content)->at('p')->replace("<p>$dom$tmp_dom</p>");
}
if ($attachments) {
my $tmp_dom = Mojo::DOM->new($tmp_attachments)->at('p')->content;
my $dom = Mojo::DOM->new($attachments)->at('p')->content;
$attachments = Mojo::DOM->new($attachments)->at('p')->replace("<p>$dom$tmp_dom</p>");
}
if ($attachments2) {
my $tmp_dom = Mojo::DOM->new($tmp_attachments2)->at('p')->content;
my $dom = Mojo::DOM->new($attachments2)->at('p')->content;
$attachments2 = Mojo::DOM->new($attachments2)->at('p')->replace("<p>$dom$tmp_dom</p>");
}
}
}
);
return ($content, $i, $date, $time, $attachments, $id, $url, $dt, $num, $attachments2);
}
}
sub extract_from_json {
my $json = shift;
## Get the content of the toot
my $content = Mojo::DOM->new($json->{content});
## Get the metadata (ie date and time) of the toot
my $dt = $f->parse_datetime($json->{created_at})->set_locale($config->{language});
$dt->set_time_zone($config->{timezone}) if defined $config->{timezone};
## Get the attachments of the toot
my $attach = c(@{$json->{media_attachments}});
return ($content, $dt, $attach);
}
sub filter_content {
my $content = shift;
## Remove style attribute
## Replace emoji with unicode characters
$content->find('img.emojione')->each(
sub {
my ($e, $num) = @_;
$e->replace($e->attr('alt'));
}
);
## Remove the configured hashtags
$content->find('a.mention.hashtag')->each(
sub {
my ($e, $num) = @_;
my $href = $e->attr('href');
$e->remove if $href =~ m#tags/($regex)$#;
}
) if $regex;
return $content;
}
sub format_time {
my $dt = shift;
## Format date and time
my $date = $dt->format_cldr($l->date_format_full);
my $time = $dt->format_cldr($l->time_format_medium);
return ($date, $time),
}
sub fetch_attachments {
my ($attach, $num) = @_;
## Store the attached images locally
my @imgs;
my @imgs2;
$attach->each(
sub {
my ($e, $inum) = @_;
my $src = $e->{url};
$src =~ s#\?(.*)##;
my $q = $1;
$src =~ m#/([^/]*)$#;
my $n = $1;
$src = Mojo::URL->new($src);
## Attachments cache
my $acache = (-e "cache/img/$n" && -e "cache/img/$n.meta");
my $msg = sprintf(" Fetching image: %s", $src);
$msg .= " from cache" if $acache;
say $msg;
my $img = "img/$n";;
my $rmime;
if ($acache) {
# Get file metadata from cache
$rmime = decode_json(Mojo::File->new("cache/$img.meta")->slurp);
# Copy file
Mojo::File->new("cache/$img")->copy_to("public/$img")->copy_to("public/epub/OPS/$img");
} else {
my $r = $ua->get($src)->result;
if ($r->is_success) {
my $body = $r->body;
$rmime = $r->headers->content_type;
# Create cache
mkdir 'cache/img' unless -d 'cache/img';
Mojo::File->new("cache/$img")->spurt($body);
Mojo::File->new("cache/$img.meta")->spurt(encode_json($rmime));
# Copy file
Mojo::File->new("public/$img")->spurt($body);
Mojo::File->new("public/epub/OPS/$img")->spurt($body);
} elsif ($r->is_error) {
die sprintf("Error while fetching %s: %s", $src, $m->dumper($r->message));
}
}
if ($e->{type} eq 'image') {
push @imgs, "<img class=\"u-max-full-width\" src=\"$img\" alt=\"\">";
push @imgs2, "<img class=\"u-max-full-width\" src=\"../$img\" alt=\"\" />";
} elsif ($e->{type} eq 'video') {
push @imgs, "<video class=\"u-max-full-width\" src=\"$img\" alt=\"\"></video>";
push @imgs2, "<video class=\"u-max-full-width\" src=\"../$img\" alt=\"\" ></video>";
}
$manifest->append_content(" <item id=\"i$num-$inum\" href=\"$img\" media-type=\"$rmime\" />\n");
}
);
my $attachments = (scalar @imgs) ? "<div><p>@imgs</p></div>" : '';
my $attachments2 = (scalar @imgs2) ? "<div><p>@imgs2</p></div>" : '';
return ($attachments, $attachments2);
}
sub process_md {
my $content = shift;
$content =~ s#%%md#%%LAST_MARKDOWN%%#g;
$content =~ s#%md##g;
$content =~ s#%%LAST_MARKDOWN%%#%md#g;
# Images
$content =~ s#!\[(.*?)\]\(<a href="([^"]*)".*?</a>\)#![$1]($2)#gm;
# Links with text
$content =~ s#\[(.*?)\]\(<a href="([^"]*)".*?</a>\)#[$1]($2)#gm;
# Links
$content =~ s#&lt;<a href="([^"]*)".*?</a>&gt;#<$1>#gm;
my $md;
my $html = encode('UTF-8', $content);
pandoc -f => 'markdown', -t => 'html', { in => \$html, out => \$md };
say $md;
$content = decode('UTF-8', $md);
return $content;
}
sub format_and_insert_toot {
my ($content, $i, $date, $time, $attachments, $id, $url, $dt, $num, $attachments2) = @_;
$content =~ s#<p></p>|</br>##g;
$content =~ s#<br>#<br />#g;
my $append = "\n <article><a href=\"#$id\"><h3 id=\"$id\">$date <span style=\"font-size: 0.5em;\">$time</span></h3></a>$content$attachments<div><a href=\"$i\"><em>Source</em></a></div></article><hr>\n";
## Insert the toot
$c->append_content($append);
## Paginate
if ($pagination && (($num % $pagination) == 0 || ($num == $urls->size))) {
my $prec = ($page == 2) ? 'index' : 'page'.($page - 1);
if ($num == $urls->size) {
$c->append_content("\n <p class=\"u-pull-left\"><a class=\"button button-primary\" href=\"$prec.html\">⇐ Page précédente</a></p>");
} else {
$page++;
if ($page == 2) {
$c->append_content("\n <p class=\"u-pull-right\"><a class=\"button button-primary\" href=\"page$page.html\">Page suivante ⇒</a></p>");
} else {
$c->append_content("\n <p class=\"u-pull-left\"><a class=\"button button-primary\" href=\"$prec.html\">⇐ Page précédente</a></p><p class=\"u-pull-right\"><a class=\"button button-primary\" href=\"page$page.html\">Page suivante ⇒</a></p>");
}
$c->append_content(" ");
$file->spurt(encode('UTF-8', $index));
$file = Mojo::File->new("public/page$page.html");
$index = open_index();
$c = $index->find('#content')->first;
$c->append_content(" <p><a href=\"".slugify($config->{title}).".epub\">Epub</a></p>\n");
}
}
## Create entries for the atom feed
push @entries, {
title => "$date $time",
id => "$url#$id",
updated => DateTime::Format::RFC3339->format_datetime($dt),
content => $append
};
## Epub
push @pages, {
num => $num,
date => $date,
time => $time,
content => "<h1>$date <span style=\"font-size: 0.5em;\">$time</span></h1><hr/>$content$attachments2<div><a href=\"$i\"><em>Source</em></a></div>"
};
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment