Verified Commit fa0ff2ce authored by Luc Didry's avatar Luc Didry
Browse files

Use CHI for cache system

This commit is dedicated to Obrow, who is supporting me with Ğ1.
Many thanks :-)
parent 477242e5
......@@ -3,8 +3,9 @@ Revision history for Perl application Lstu
0.12 ????-??-??
- Improve LDAP User Authentication Support (beware of configuration
changes!) <Ira W. Snyder>
- Allow users to configure maximum number of database connections
- Allow users to configure maximum number of database connections
<Ira W. Snyder>
- Use CHI for cache system
- Update dependencies
- Fix bugs
......
......@@ -21,3 +21,5 @@ requires 'IO::Socket::SSL';
requires 'Net::LDAP';
requires 'Apache::Htpasswd';
requires 'Image::PNG::QRCode';
requires 'CHI';
requires 'Data::Serializer';
......@@ -30,6 +30,78 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 6.42
Test::More 0
perl 5.005
CHI-0.60
pathname: J/JS/JSWARTZ/CHI-0.60.tar.gz
provides:
CHI 0.60
CHI::CacheObject 0.60
CHI::Driver 0.60
CHI::Driver::Base::CacheContainer 0.60
CHI::Driver::CacheCache 0.60
CHI::Driver::FastMmap 0.60
CHI::Driver::File 0.60
CHI::Driver::Memory 0.60
CHI::Driver::Metacache 0.60
CHI::Driver::Null 0.60
CHI::Driver::RawMemory 0.60
CHI::Driver::Role::HasSubcaches 0.60
CHI::Driver::Role::IsSizeAware 0.60
CHI::Driver::Role::IsSubcache 0.60
CHI::Stats 0.60
requirements:
Carp::Assert 0.20
Class::Load 0
Data::UUID 0
Digest::JHash 0
Digest::MD5 0
ExtUtils::MakeMaker 0
File::Spec 0.80
Hash::MoreUtils 0
JSON::MaybeXS 1.003003
List::MoreUtils 0.13
Log::Any 0.08
Moo 1.003
MooX::Types::MooseLike 0.23
MooX::Types::MooseLike::Base 0
MooX::Types::MooseLike::Numeric 0
Storable 0
String::RewritePrefix 0
Task::Weaken 0
Time::Duration 1.06
Time::Duration::Parse 0.03
Time::HiRes 1.30
Try::Tiny 0.05
Carp-Assert-0.21
pathname: N/NE/NEILB/Carp-Assert-0.21.tar.gz
provides:
Carp::Assert 0.21
requirements:
Carp 0
Exporter 0
ExtUtils::MakeMaker 0
perl 5.006
strict 0
vars 0
warnings 0
Class-Load-0.24
pathname: E/ET/ETHER/Class-Load-0.24.tar.gz
provides:
Class::Load 0.24
Class::Load::PP 0.24
requirements:
Carp 0
Data::OptList 0
Exporter 0
ExtUtils::MakeMaker 0
Module::Implementation 0.04
Module::Runtime 0.012
Package::Stash 0.14
Scalar::Util 0
Try::Tiny 0
base 0
perl 5.006
strict 0
warnings 0
Class-Method-Modifiers-2.12
pathname: E/ET/ETHER/Class-Method-Modifiers-2.12.tar.gz
provides:
......@@ -207,6 +279,55 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 6.48
Test::Simple 0.90
perl 5.008
Data-OptList-0.110
pathname: R/RJ/RJBS/Data-OptList-0.110.tar.gz
provides:
Data::OptList 0.110
requirements:
ExtUtils::MakeMaker 0
List::Util 0
Params::Util 0
Sub::Install 0.921
strict 0
warnings 0
Data-Serializer-0.60
pathname: N/NE/NEELY/Data-Serializer-0.60.tar.gz
provides:
Data::Serializer 0.60
Data::Serializer::Bencode 0.03
Data::Serializer::Config::General 0.02
Data::Serializer::Convert::Bencode 0.03
Data::Serializer::Convert::Bencode_XS 0.03
Data::Serializer::Cookbook 0.05
Data::Serializer::Data::Denter 0.02
Data::Serializer::Data::Dumper 0.05
Data::Serializer::Data::Taxi 0.02
Data::Serializer::FreezeThaw 0.02
Data::Serializer::JSON 0.04
Data::Serializer::JSON::Syck 0.02
Data::Serializer::PHP::Serialization 0.02
Data::Serializer::Persistent 0.01
Data::Serializer::Raw 0.02
Data::Serializer::Storable 0.03
Data::Serializer::XML::Dumper 0.02
Data::Serializer::XML::Simple 0.03
Data::Serializer::YAML 0.02
Data::Serializer::YAML::Syck 0.02
requirements:
AutoLoader 0
Data::Dumper 2.08
Digest::SHA 0
Exporter 0
File::Spec 0
IO::File 0
Test::More 0
Data-UUID-1.221
pathname: R/RJ/RJBS/Data-UUID-1.221.tar.gz
provides:
Data::UUID 1.221
requirements:
Digest::MD5 0
ExtUtils::MakeMaker 0
Data-Validate-Domain-0.14
pathname: D/DR/DROLSKY/Data-Validate-Domain-0.14.tar.gz
provides:
......@@ -257,6 +378,30 @@ DISTRIBUTIONS
Digest::SHA 1
ExtUtils::MakeMaker 0
perl 5.004
Digest-JHash-0.10
pathname: S/SH/SHLOMIF/Digest-JHash-0.10.tar.gz
provides:
Digest::JHash 0.10
requirements:
DynaLoader 0
Exporter 0
ExtUtils::MakeMaker 0
perl 5.008
strict 0
vars 0
warnings 0
Dist-CheckConflicts-0.11
pathname: D/DO/DOY/Dist-CheckConflicts-0.11.tar.gz
provides:
Dist::CheckConflicts 0.11
requirements:
Carp 0
Exporter 0
ExtUtils::MakeMaker 6.30
Module::Runtime 0.009
base 0
strict 0
warnings 0
Encode-Locale-1.05
pathname: G/GA/GAAS/Encode-Locale-1.05.tar.gz
provides:
......@@ -266,6 +411,24 @@ DISTRIBUTIONS
Encode::Alias 0
ExtUtils::MakeMaker 0
perl 5.008
Exporter-Lite-0.08
pathname: N/NE/NEILB/Exporter-Lite-0.08.tar.gz
provides:
Exporter::Lite 0.08
requirements:
Carp 0
ExtUtils::MakeMaker 6.3
perl 5.006
strict 0
warnings 0
Exporter-Tiny-1.000000
pathname: T/TO/TOBYINK/Exporter-Tiny-1.000000.tar.gz
provides:
Exporter::Shiny 1.000000
Exporter::Tiny 1.000000
requirements:
ExtUtils::MakeMaker 6.17
perl 5.006001
ExtUtils-Config-0.008
pathname: L/LE/LEONT/ExtUtils-Config-0.008.tar.gz
provides:
......@@ -438,6 +601,12 @@ DISTRIBUTIONS
Clone 0
ExtUtils::MakeMaker 0
perl 5.008001
Hash-MoreUtils-0.05
pathname: R/RE/REHSACK/Hash-MoreUtils-0.05.tar.gz
provides:
Hash::MoreUtils 0.05
requirements:
Test::More 0.90
IO-HTML-1.001
pathname: C/CJ/CJM/IO-HTML-1.001.tar.gz
provides:
......@@ -488,6 +657,16 @@ DISTRIBUTIONS
requirements:
ExtUtils::MakeMaker 0
Test::More 0
JSON-MaybeXS-1.004000
pathname: H/HA/HAARG/JSON-MaybeXS-1.004000.tar.gz
provides:
JSON::MaybeXS 1.004000
requirements:
Carp 0
ExtUtils::MakeMaker 0
JSON::PP 2.27300
Scalar::Util 0
perl 5.006
LWP-MediaTypes-6.02
pathname: G/GA/GAAS/LWP-MediaTypes-6.02.tar.gz
provides:
......@@ -495,6 +674,29 @@ DISTRIBUTIONS
requirements:
ExtUtils::MakeMaker 0
perl 5.006002
List-MoreUtils-0.428
pathname: R/RE/REHSACK/List-MoreUtils-0.428.tar.gz
provides:
List::MoreUtils 0.428
List::MoreUtils::PP 0.428
requirements:
Exporter::Tiny 0.038
ExtUtils::MakeMaker 0
List::MoreUtils::XS 0.426
List-MoreUtils-XS-0.428
pathname: R/RE/REHSACK/List-MoreUtils-XS-0.428.tar.gz
provides:
List::MoreUtils::XS 0.428
requirements:
Carp 0
ExtUtils::MakeMaker 0
File::Basename 0
File::Copy 0
File::Path 0
File::Spec 0
IPC::Cmd 0
XSLoader 0.22
base 0
Locale-Maketext-Lexicon-1.00
pathname: D/DR/DRTECH/Locale-Maketext-Lexicon-1.00.tar.gz
provides:
......@@ -523,6 +725,40 @@ DISTRIBUTIONS
requirements:
ExtUtils::MakeMaker 6.30
Locale::Maketext 1.17
Log-Any-1.705
pathname: P/PR/PREACTION/Log-Any-1.705.tar.gz
provides:
Log::Any 1.705
Log::Any::Adapter 1.705
Log::Any::Adapter::Base 1.705
Log::Any::Adapter::File 1.705
Log::Any::Adapter::Null 1.705
Log::Any::Adapter::Stderr 1.705
Log::Any::Adapter::Stdout 1.705
Log::Any::Adapter::Syslog 1.705
Log::Any::Adapter::Test 1.705
Log::Any::Adapter::Util 1.705
Log::Any::Manager 1.705
Log::Any::Proxy 1.705
Log::Any::Proxy::Null 1.705
Log::Any::Proxy::Test 1.705
Log::Any::Test 1.705
requirements:
B 0
Carp 0
Data::Dumper 0
Exporter 0
ExtUtils::MakeMaker 0
Fcntl 0
File::Basename 0
FindBin 0
IO::File 0
Storable 0
Sys::Syslog 0
Test::Builder 0
constant 0
strict 0
warnings 0
MRO-Compat-0.12
pathname: B/BO/BOBTFISH/MRO-Compat-0.12.tar.gz
provides:
......@@ -640,6 +876,17 @@ DISTRIBUTIONS
perl 5.006
strict 0
warnings 0
Module-Implementation-0.09
pathname: D/DR/DROLSKY/Module-Implementation-0.09.tar.gz
provides:
Module::Implementation 0.09
requirements:
Carp 0
ExtUtils::MakeMaker 0
Module::Runtime 0.012
Try::Tiny 0
strict 0
warnings 0
Module-Install-1.19
pathname: E/ET/ETHER/Module-Install-1.19.tar.gz
provides:
......@@ -970,6 +1217,24 @@ DISTRIBUTIONS
Sub::Defer 2.003001
Sub::Quote 2.003001
perl 5.006
MooX-Types-MooseLike-0.29
pathname: M/MA/MATEU/MooX-Types-MooseLike-0.29.tar.gz
provides:
MooX::Types::MooseLike 0.29
MooX::Types::MooseLike::Base 0.29
requirements:
ExtUtils::MakeMaker 0
Module::Runtime 0.014
MooX-Types-MooseLike-Numeric-1.03
pathname: M/MA/MATEU/MooX-Types-MooseLike-Numeric-1.03.tar.gz
provides:
MooX::Types::MooseLike::Numeric 1.03
requirements:
ExtUtils::MakeMaker 0
Moo 1.004002
MooX::Types::MooseLike 0.23
Test::Fatal 0.003
Test::More 0.96
Mozilla-CA-20160104
pathname: A/AB/ABH/Mozilla-CA-20160104.tar.gz
provides:
......@@ -1165,6 +1430,36 @@ DISTRIBUTIONS
Test::More 0.47
Test::Script 1.06
perl 5.006
Package-Stash-0.37
pathname: D/DO/DOY/Package-Stash-0.37.tar.gz
provides:
Package::Stash 0.37
Package::Stash::PP 0.37
requirements:
B 0
Carp 0
Config 0
Dist::CheckConflicts 0.02
ExtUtils::MakeMaker 0
File::Spec 0
Getopt::Long 0
Module::Implementation 0.06
Package::Stash::XS 0.26
Scalar::Util 0
Symbol 0
Text::ParseWords 0
constant 0
strict 0
warnings 0
Package-Stash-XS-0.28
pathname: D/DO/DOY/Package-Stash-XS-0.28.tar.gz
provides:
Package::Stash::XS 0.28
requirements:
ExtUtils::MakeMaker 6.30
XSLoader 0
strict 0
warnings 0
Params-Util-1.07
pathname: A/AD/ADAMK/Params-Util-1.07.tar.gz
provides:
......@@ -1210,12 +1505,46 @@ DISTRIBUTIONS
Sub::Quote 2.000001
Text::Balanced 2.00
perl 5.006
String-RewritePrefix-0.007
pathname: R/RJ/RJBS/String-RewritePrefix-0.007.tar.gz
provides:
String::RewritePrefix 0.007
requirements:
Carp 0
ExtUtils::MakeMaker 6.30
Sub::Exporter 0.972
strict 0
warnings 0
Sub-Exporter-0.987
pathname: R/RJ/RJBS/Sub-Exporter-0.987.tar.gz
provides:
Sub::Exporter 0.987
Sub::Exporter::Util 0.987
requirements:
Carp 0
Data::OptList 0.100
ExtUtils::MakeMaker 6.30
Params::Util 0.14
Sub::Install 0.92
strict 0
warnings 0
Sub-Exporter-Progressive-0.001013
pathname: F/FR/FREW/Sub-Exporter-Progressive-0.001013.tar.gz
provides:
Sub::Exporter::Progressive 0.001013
requirements:
ExtUtils::MakeMaker 0
Sub-Install-0.928
pathname: R/RJ/RJBS/Sub-Install-0.928.tar.gz
provides:
Sub::Install 0.928
requirements:
B 0
Carp 0
ExtUtils::MakeMaker 6.30
Scalar::Util 0
strict 0
warnings 0
Sub-Quote-2.003001
pathname: H/HA/HAARG/Sub-Quote-2.003001.tar.gz
provides:
......@@ -1225,6 +1554,29 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 0
Scalar::Util 0
perl 5.006
Task-Weaken-1.06
pathname: E/ET/ETHER/Task-Weaken-1.06.tar.gz
provides:
Task::Weaken 1.06
requirements:
Config 0
ExtUtils::MakeMaker 0
File::Spec 0
Scalar::Util 1.14
perl 5.006
strict 0
Test-Fatal-0.014
pathname: R/RJ/RJBS/Test-Fatal-0.014.tar.gz
provides:
Test::Fatal 0.014
requirements:
Carp 0
Exporter 5.57
ExtUtils::MakeMaker 0
Test::Builder 0
Try::Tiny 0.07
strict 0
warnings 0
Test-Script-1.15
pathname: P/PL/PLICEASE/Test-Script-1.15.tar.gz
provides:
......@@ -1244,6 +1596,28 @@ DISTRIBUTIONS
requirements:
ExtUtils::MakeMaker 0
if 0
Time-Duration-1.20
pathname: N/NE/NEILB/Time-Duration-1.20.tar.gz
provides:
Time::Duration 1.20
requirements:
Exporter 0
ExtUtils::MakeMaker 0
constant 0
perl 5.006
strict 0
warnings 0
Time-Duration-Parse-0.13
pathname: N/NE/NEILB/Time-Duration-Parse-0.13.tar.gz
provides:
Time::Duration::Parse 0.13
requirements:
Carp 0
Exporter::Lite 0
ExtUtils::MakeMaker 0
perl 5.006
strict 0
warnings 0
Try-Tiny-0.28
pathname: E/ET/ETHER/Try-Tiny-0.28.tar.gz
provides:
......
......@@ -5,6 +5,7 @@ use Mojo::JSON;
use Net::LDAP;
use Apache::Htpasswd;
use Lstu::DB::URL;
use CHI;
$ENV{MOJO_REVERSE_PROXY} = 1;
......@@ -30,6 +31,7 @@ sub startup {
dbtype => 'sqlite',
max_redir => 2,
skip_spamhaus => 0,
cache_max_size => 2,
}
});
......@@ -53,6 +55,16 @@ sub startup {
eval qq(use lib "$lib");
$self->plugin('I18N');
# Cache
my $cache_max_size = ($config->{cache_max_size} > 0) ? 8 * 1024 * 1024 * $config->{cache_max_size} : 1;
$self->{urls_cache} = CHI->new(
driver => 'Memory',
global => 1,
is_size_aware => 1,
max_size => $cache_max_size,
expires_in => '1 day'
);
# Debug
$self->plugin('DebugDumperHelper');
......
......@@ -287,16 +287,11 @@ sub get {
my $c = shift;
my $short = $c->param('short');
my ($url, $db_url);
if (defined $c->cache->{$short}) {
$url = $c->cache->{$short}->{url};
} else {
$db_url = Lstu::DB::URL->new(
app => $c,
short => $short
);
$url = $db_url->url;
}
my $url = $c->app->{urls_cache}->compute($short, undef, sub {
return Lstu::DB::URL->new(app => $c, short => $short)->url;
});
$c->debug($c->app->{urls_cache}->get_keys( ));
if ($url) {
$c->respond_to(
json => { json => { success => Mojo::JSON->true, url => $url } },
......@@ -308,20 +303,10 @@ sub get {
);
# Update counter
$c->on(finish => sub {
$c->cache->{$short} = {
last_used => time,
url => $url
};
if ($c->config('minion')->{enabled} && $c->config('minion')->{db_path}) {
$c->app->minion->enqueue(increase_counter => [$short, $c->{url}]);
} else {
$db_url = Lstu::DB::URL->new(
app => $c,
short => $short
) if (defined $c->cache->{$short});
$db_url->increment_counter;
Lstu::DB::URL->new(app => $c, short => $short)->increment_counter;
my $piwik = $c->config('piwik');
if (defined($piwik) && $piwik->{idsite} && $piwik->{url}) {
......@@ -335,7 +320,6 @@ sub get {
}
}
$c->clear_cache;
});
} else {
my $msg = $c->l('The shortened URL %1 doesn\'t exist.', $c->url_for('/')->to_abs.$short);
......
......@@ -33,8 +33,6 @@ sub register {
}
}
$app->helper(cache => \&_cache);
$app->helper(clear_cache => \&_clear_cache);
$app->helper(ip => \&_ip);
$app->helper(provisioning => \&_provisioning);
$app->helper(prefix => \&_prefix);
......@@ -71,22 +69,6 @@ sub _mysql {
return $mysql;
}
sub _cache {
my $c = shift;
state $cache = {};
}
sub _clear_cache {
my $c = shift;
my $cache = $c->cache;
my @keys = keys %{$cache};
my $limit = ($c->app->mode eq 'production') ? 500 : 1;
map {delete $cache->{$_};} @keys if (scalar(@keys) > $limit);
}
sub _ip {
my $c = shift;
......
......@@ -180,4 +180,15 @@
# set to 1 to skip SpamHaus check (not recommended)
# optional, default is 0
#skip_spamhaus => 0,
# maximum size (in MB) of memory allowed for the URL cache
# Lstu has a built-in memory-based URL cache to accelerate responses to often-viewed URL.
# This setting makes the cache remove oldest viewed image if the cache size is over it.
# WARNING: a cache is created for each hypnotoad worker, which by default is twice the number of
# CPUs you have. See http://mojolicious.org/perldoc/Mojo/Server/Hypnotoad#workers for details
# So, if you have 4 workers and set cache_max_size to 10, the real maximum size of RAM used for
# cache is 40MB.
# If set to 0, the cache is disabled
# optional, default is 2
#cache_max_size => 2,
};
......@@ -154,7 +154,7 @@ msgstr ""
msgid "Password"
msgstr ""
#: lib/Lstu.pm:275 themes/default/templates/api.html.ep:23
#: lib/Lstu.pm:287 themes/default/templates/api.html.ep:23
msgid "Please, check your credentials: unable to authenticate."
msgstr ""
......@@ -195,22 +195,22 @@ msgid "Statistics"
msgstr ""
#. ($url->host)
#: lib/Lstu/Plugin/Helpers.pm:161
#: lib/Lstu/Plugin/Helpers.pm:143
msgid "The URL host or one of its redirection(s) (%1) is blacklisted at Spamhaus. I refuse to shorten it."
msgstr ""
#. ($c->config('max_redir')
#: lib/Lstu/Plugin/Helpers.pm:177
#: lib/Lstu/Plugin/Helpers.pm:159
msgid "The URL redirects %1 times or most. It's most likely a dangerous URL (spam, phishing, etc.). I refuse to shorten it."
msgstr ""
#. ($url->host)
#: lib/Lstu/Plugin/Helpers.pm:153
#: lib/Lstu/Plugin/Helpers.pm:135
msgid "The URL you want to shorten comes from a domain (%1) that is blacklisted on this server (usually because of spammers that use this domain)."
msgstr ""
#. ($c->url_for('/')
#: lib/Lstu/Controller/Admin.pm:131 lib/Lstu/Controller/Lstu.pm:281 lib/Lstu/Controller/Lstu.pm:341