Commit bb097294 authored by Luc Didry's avatar Luc Didry

Built-in image cache system

parent cfab86c4
Pipeline #43146 passed with stages
in 5 minutes 32 seconds
......@@ -3,6 +3,7 @@ Revision history for Lutim
????? ??????????
- PostgreSQL performance improvments
- Move some tasks to recurring instead of being in after_dispatch hook
- Built-in image cache system \o/
0.9.6 2018-03-12
- Update translations
......
......@@ -27,3 +27,5 @@ requires 'Image::ExifTool';
requires 'Data::Entropy';
requires 'List::MoreUtils', '> 0.33';
requires 'Archive::Zip';
requires 'CHI';
requires 'Data::Serializer';
This diff is collapsed.
......@@ -3,6 +3,7 @@ package Lutim;
use Mojo::Base 'Mojolicious';
use Mojo::IOLoop;
use Lutim::DB::Image;
use CHI;
use vars qw($im_loaded);
BEGIN {
......@@ -52,9 +53,19 @@ sub startup {
dbtype => 'sqlite',
db_path => 'minion.db'
},
cache_max_size => 0,
}
});
my $cache_max_size = ($config->{cache_max_size} > 0) ? 8 * 1024 * 1024 * $config->{cache_max_size} : 1;
$self->{images_cache} = CHI->new(
driver => 'Memory',
global => 1,
is_size_aware => 1,
max_size => $cache_max_size,
expires_in => '1 day'
);
die "You need to provide a contact information in lutim.conf !" unless (defined($config->{contact}));
$ENV{MOJO_MAX_MESSAGE_SIZE} = $config->{max_file_size};
......
......@@ -2,6 +2,7 @@
package Lutim::Plugin::Helpers;
use Mojo::Base 'Mojolicious::Plugin';
use Mojo::Util qw(quote);
use Mojo::File;
use Crypt::CBC;
use Data::Entropy qw(entropy_source);
use DateTime;
......@@ -77,7 +78,6 @@ sub _render_file {
$dl = 'attachment' if ($mediatype =~ m/svg/);
$filename = quote($filename);
my $asset;
unless (-f $path && -r $path) {
$c->app->log->error("Cannot read file [$path]. error [$!]");
$c->flash(
......@@ -98,11 +98,30 @@ sub _render_file {
$headers->add('Content-Disposition' => $dl.';filename='.$filename);
$c->res->content->headers($headers);
if ($key) {
$asset = $c->decrypt($key, $path, $iv);
} else {
$asset = Mojo::Asset::File->new(path => $path);
my $cache = $c->app->{images_cache}->compute($img->short, undef, sub {
if ($key) {
return {
asset => $c->decrypt($key, $path, $iv),
key => $key
};
} else {
return {
asset => Mojo::File->new($path)->slurp,
};
}
});
if ($key && $key ne $cache->{key}) {
$c->app->{images_cache}->replace(
$img->short,
{
asset => $c->decrypt($key, $path, $iv),
key => $key
},
);
}
# Extend expiration time
my $asset = Mojo::Asset::Memory->new;
$asset->add_chunk($cache->{asset});
if (defined $thumb && $im_loaded && $mediatype ne 'image/svg+xml' && $mediatype !~ m#image/(x-)?xcf# && $mediatype ne 'image/webp') { # ImageMagick don't work in Debian with svg (for now?)
my $im = Image::Magick->new;
......@@ -261,17 +280,20 @@ sub _decrypt {
open(my $f, "<",$file) or die "Unable to read encrypted file: $!";
binmode $f;
while (read($f, my $buffer,1024)) {
while (read($f, my $buffer, 1024)) {
$decrypt_asset->add_chunk($cipher->crypt($buffer));
}
$decrypt_asset->add_chunk($cipher->finish) ;
return $decrypt_asset;
return $decrypt_asset->slurp;
}
sub _delete_image {
my $c = shift;
my $img = shift;
if ($c->app->{images_cache}) {
$c->app->{images_cache}->remove($img->short);
}
unlink $img->path or warn "Could not unlink ".$img->path.": $!";
$img->disable();
}
......
......@@ -46,6 +46,7 @@ sub startup {
dbtype => 'sqlite',
db_path => 'minion.db'
},
cache_max_size => 0,
}
}
);
......
......@@ -92,7 +92,7 @@
# comma-separated values proposed for delays
# optional, default is '0,1,7,30,365'
#proposed_delays => '0,1,7,30,365',
# number of days after which the images will be deleted, even if they were uploaded with "no delay" (or value superior to max_delay)
# a warning message will be displayed on homepage
# optional, default is 0 (no limit)
......@@ -175,6 +175,13 @@
# optional, default is 15
#max_files_in_zip => 15,
# maximum size (in MB) of memory allowed for the image cache
# Lutim has a built-in memory-based image cache to accelerate responses to often-viewed images.
# This setting makes the cache remove oldest viewed image if the cache size is over it.
# If set to 0, the cache is disabled
# optional, default is 0
#cache_max_size => 0,
##########################
# Lutim cron jobs settings
##########################
......
......@@ -89,6 +89,10 @@
# optional, default is 0 (no limit)
default_delay => 30,
# comma-separated values proposed for delays
# optional, default is '0,1,7,30,365'
#proposed_delays => '0,1,7,30,365',
# number of days after which the images will be deleted, even if they were uploaded with "no delay" (or value superior to max_delay)
# a warning message will be displayed on homepage
# optional, default is 0 (no limit)
......@@ -171,6 +175,13 @@
# optional, default is 15
#max_files_in_zip => 15,
# maximum size (in MB) of memory allowed for the image cache
# Lutim has a built-in memory-based image cache to accelerate responses to often-viewed images.
# This setting makes the cache remove oldest viewed image if the cache size is over it.
# If set to 0, the cache is disabled
# optional, default is 0
#cache_max_size => 0,
##########################
# Lutim cron jobs settings
##########################
......
......@@ -89,6 +89,10 @@
# optional, default is 0 (no limit)
default_delay => 30,
# comma-separated values proposed for delays
# optional, default is '0,1,7,30,365'
#proposed_delays => '0,1,7,30,365',
# number of days after which the images will be deleted, even if they were uploaded with "no delay" (or value superior to max_delay)
# a warning message will be displayed on homepage
# optional, default is 0 (no limit)
......@@ -171,6 +175,13 @@
# optional, default is 15
#max_files_in_zip => 15,
# maximum size (in MB) of memory allowed for the image cache
# Lutim has a built-in memory-based image cache to accelerate responses to often-viewed images.
# This setting makes the cache remove oldest viewed image if the cache size is over it.
# If set to 0, the cache is disabled
# optional, default is 0
#cache_max_size => 0,
##########################
# Lutim cron jobs settings
##########################
......
......@@ -89,6 +89,10 @@
# optional, default is 0 (no limit)
default_delay => 30,
# comma-separated values proposed for delays
# optional, default is '0,1,7,30,365'
#proposed_delays => '0,1,7,30,365',
# number of days after which the images will be deleted, even if they were uploaded with "no delay" (or value superior to max_delay)
# a warning message will be displayed on homepage
# optional, default is 0 (no limit)
......@@ -171,6 +175,13 @@
# optional, default is 15
#max_files_in_zip => 15,
# maximum size (in MB) of memory allowed for the image cache
# Lutim has a built-in memory-based image cache to accelerate responses to often-viewed images.
# This setting makes the cache remove oldest viewed image if the cache size is over it.
# If set to 0, the cache is disabled
# optional, default is 0
#cache_max_size => 0,
##########################
# Lutim cron jobs settings
##########################
......
......@@ -89,6 +89,10 @@
# optional, default is 0 (no limit)
default_delay => 30,
# comma-separated values proposed for delays
# optional, default is '0,1,7,30,365'
#proposed_delays => '0,1,7,30,365',
# number of days after which the images will be deleted, even if they were uploaded with "no delay" (or value superior to max_delay)
# a warning message will be displayed on homepage
# optional, default is 0 (no limit)
......@@ -171,6 +175,13 @@
# optional, default is 15
#max_files_in_zip => 15,
# maximum size (in MB) of memory allowed for the image cache
# Lutim has a built-in memory-based image cache to accelerate responses to often-viewed images.
# This setting makes the cache remove oldest viewed image if the cache size is over it.
# If set to 0, the cache is disabled
# optional, default is 0
#cache_max_size => 0,
##########################
# Lutim cron jobs settings
##########################
......
......@@ -89,6 +89,10 @@
# optional, default is 0 (no limit)
default_delay => 30,
# comma-separated values proposed for delays
# optional, default is '0,1,7,30,365'
#proposed_delays => '0,1,7,30,365',
# number of days after which the images will be deleted, even if they were uploaded with "no delay" (or value superior to max_delay)
# a warning message will be displayed on homepage
# optional, default is 0 (no limit)
......@@ -171,6 +175,13 @@
# optional, default is 15
#max_files_in_zip => 15,
# maximum size (in MB) of memory allowed for the image cache
# Lutim has a built-in memory-based image cache to accelerate responses to often-viewed images.
# This setting makes the cache remove oldest viewed image if the cache size is over it.
# If set to 0, the cache is disabled
# optional, default is 0
#cache_max_size => 0,
##########################
# Lutim cron jobs settings
##########################
......
......@@ -89,6 +89,10 @@
# optional, default is 0 (no limit)
default_delay => 30,
# comma-separated values proposed for delays
# optional, default is '0,1,7,30,365'
#proposed_delays => '0,1,7,30,365',
# number of days after which the images will be deleted, even if they were uploaded with "no delay" (or value superior to max_delay)
# a warning message will be displayed on homepage
# optional, default is 0 (no limit)
......@@ -171,6 +175,13 @@
# optional, default is 15
#max_files_in_zip => 15,
# maximum size (in MB) of memory allowed for the image cache
# Lutim has a built-in memory-based image cache to accelerate responses to often-viewed images.
# This setting makes the cache remove oldest viewed image if the cache size is over it.
# If set to 0, the cache is disabled
# optional, default is 0
#cache_max_size => 0,
##########################
# Lutim cron jobs settings
##########################
......
......@@ -26,12 +26,35 @@ BEGIN {
{
file => $cfile->to_abs->to_string,
default => {
dbtype => 'sqlite'
provisioning => 100,
provis_step => 5,
length => 8,
always_encrypt => 0,
anti_flood_delay => 5,
tweet_card_via => '@framasky',
max_file_size => 10*1024*1024,
https => 0,
proposed_delays => '0,1,7,30,365',
default_delay => 0,
max_delay => 0,
token_length => 24,
crypto_key_length => 8,
thumbnail_size => 100,
theme => 'default',
dbtype => 'sqlite',
db_path => 'lutim.db',
max_files_in_zip => 15,
prefix => '/',
minion => {
enabled => 0,
dbtype => 'sqlite',
db_path => 'minion.db'
},
cache_max_size => 0,
}
}
);
$m->plugin('Lutim::Plugin::Helpers');
$m->plugin('DebugDumperHelper');
}
# Home page
......
......@@ -32,11 +32,11 @@ msgstr ""
msgid "-or-"
msgstr ""
#: lib/Lutim.pm:188 lib/Lutim/Command/cron/stats.pm:151 lib/Lutim/Command/cron/stats.pm:162 lib/Lutim/Command/cron/stats.pm:179 themes/default/templates/index.html.ep:5 themes/default/templates/raw.html.ep:10 themes/default/templates/raw.html.ep:21 themes/default/templates/raw.html.ep:38
#: lib/Lutim.pm:199 lib/Lutim/Command/cron/stats.pm:151 lib/Lutim/Command/cron/stats.pm:162 lib/Lutim/Command/cron/stats.pm:179 themes/default/templates/index.html.ep:5 themes/default/templates/raw.html.ep:10 themes/default/templates/raw.html.ep:21 themes/default/templates/raw.html.ep:38
msgid "1 year"
msgstr ""
#: lib/Lutim.pm:187 lib/Lutim/Command/cron/stats.pm:148 lib/Lutim/Command/cron/stats.pm:159 lib/Lutim/Command/cron/stats.pm:176 themes/default/templates/index.html.ep:4 themes/default/templates/partial/for_my_delay.html.ep:13 themes/default/templates/partial/lutim.js.ep:151 themes/default/templates/raw.html.ep:18 themes/default/templates/raw.html.ep:35 themes/default/templates/raw.html.ep:7
#: lib/Lutim.pm:198 lib/Lutim/Command/cron/stats.pm:148 lib/Lutim/Command/cron/stats.pm:159 lib/Lutim/Command/cron/stats.pm:176 themes/default/templates/index.html.ep:4 themes/default/templates/partial/for_my_delay.html.ep:13 themes/default/templates/partial/lutim.js.ep:151 themes/default/templates/raw.html.ep:18 themes/default/templates/raw.html.ep:35 themes/default/templates/raw.html.ep:7
msgid "24 hours"
msgstr ""
......@@ -443,7 +443,7 @@ msgid "Uploaded files by days"
msgstr ""
#. ($c->app->config('contact')
#: lib/Lutim/Plugin/Helpers.pm:183
#: lib/Lutim/Plugin/Helpers.pm:202
msgid "Uploading is currently disabled, please try later or contact the administrator (%1)."
msgstr ""
......@@ -495,7 +495,7 @@ msgstr ""
msgid "core developer"
msgstr ""
#: lib/Lutim.pm:186 lib/Lutim/Command/cron/stats.pm:147 lib/Lutim/Command/cron/stats.pm:158 lib/Lutim/Command/cron/stats.pm:175 themes/default/templates/index.html.ep:3 themes/default/templates/raw.html.ep:17 themes/default/templates/raw.html.ep:34 themes/default/templates/raw.html.ep:6
#: lib/Lutim.pm:197 lib/Lutim/Command/cron/stats.pm:147 lib/Lutim/Command/cron/stats.pm:158 lib/Lutim/Command/cron/stats.pm:175 themes/default/templates/index.html.ep:3 themes/default/templates/raw.html.ep:17 themes/default/templates/raw.html.ep:34 themes/default/templates/raw.html.ep:6
msgid "no time limit"
msgstr ""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment