Commit 0ab2abf6 authored by Luc Didry's avatar Luc Didry

Add Minion option for async counters increasing

This commit is dedicated to guilhemB, thanks for supporting me on Tipeee
:-)
parent ab869744
local/* local/*
.tx/* .tx/*
*.db *.db
*.db-shm
*.db-wal
*.db-journal
lstu.conf lstu.conf
*.swp *.swp
script/hypnotoad.pid script/hypnotoad.pid
......
...@@ -12,6 +12,10 @@ test: ...@@ -12,6 +12,10 @@ test:
- sed -i -e "s@ \(adminpwd.*\)@ \#\1@" -e "s@#hashed_adminpwd.*@hashed_adminpwd => '31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66',@" lstu.conf - sed -i -e "s@ \(adminpwd.*\)@ \#\1@" -e "s@#hashed_adminpwd.*@hashed_adminpwd => '31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66',@" lstu.conf
- rm -f test.db - rm -f test.db
- make test - make test
- sed -i -e "s@#minion.*@minion => { enabled => 0, db_path => 'minion.db' },@" lstu.conf
- rm -f test.db minion.db
- make minion &
- make test
tags: tags:
- Debian - Debian
- Jessie - Jessie
Revision history for Perl application Lstu Revision history for Perl application Lstu
0.08 ?
- Add Piwik tracking option
- Add Minion option for async counters increasing
0.07 Tue Dec 20 17:55:00 2016 +0100 0.07 Tue Dec 20 17:55:00 2016 +0100
- Add logo - Add logo
- Add theme system - Add theme system
......
...@@ -21,6 +21,9 @@ dev: ...@@ -21,6 +21,9 @@ dev:
devlog: devlog:
multitail log/development.log multitail log/development.log
minion:
$(CARTON) $(REAL_LSTU) minion worker
installdeps: installdeps:
carton install carton install
......
...@@ -5,6 +5,8 @@ requires 'Net::Domain::TLD', '>= 1.74'; ...@@ -5,6 +5,8 @@ requires 'Net::Domain::TLD', '>= 1.74';
requires 'Mojolicious::Plugin::I18N'; requires 'Mojolicious::Plugin::I18N';
requires 'Mojolicious::Plugin::DebugDumperHelper'; requires 'Mojolicious::Plugin::DebugDumperHelper';
requires 'Mojolicious::Plugin::Piwik'; requires 'Mojolicious::Plugin::Piwik';
requires 'Minion';
requires 'Minion::Backend::SQLite';
requires 'Locale::Maketext'; requires 'Locale::Maketext';
requires 'Locale::Maketext::Extract'; requires 'Locale::Maketext::Extract';
requires 'Net::Abuse::Utils::Spamhaus'; requires 'Net::Abuse::Utils::Spamhaus';
......
# carton snapshot format: version 1.0 # carton snapshot format: version 1.0
DISTRIBUTIONS DISTRIBUTIONS
DBD-SQLite-1.48 DBD-SQLite-1.54
pathname: I/IS/ISHIGAKI/DBD-SQLite-1.48.tar.gz pathname: I/IS/ISHIGAKI/DBD-SQLite-1.54.tar.gz
provides: provides:
DBD::SQLite 1.48 DBD::SQLite 1.54
DBD::SQLite::Constants undef DBD::SQLite::Constants undef
DBD::SQLite::VirtualTable 1.48 DBD::SQLite::VirtualTable 1.54
DBD::SQLite::VirtualTable::Cursor 1.48 DBD::SQLite::VirtualTable::Cursor 1.54
DBD::SQLite::VirtualTable::FileContent undef DBD::SQLite::VirtualTable::FileContent undef
DBD::SQLite::VirtualTable::FileContent::Cursor undef DBD::SQLite::VirtualTable::FileContent::Cursor undef
DBD::SQLite::VirtualTable::PerlData undef DBD::SQLite::VirtualTable::PerlData undef
...@@ -156,6 +156,45 @@ DISTRIBUTIONS ...@@ -156,6 +156,45 @@ DISTRIBUTIONS
Digest::SHA 1 Digest::SHA 1
ExtUtils::MakeMaker 0 ExtUtils::MakeMaker 0
perl 5.004 perl 5.004
ExtUtils-Config-0.008
pathname: L/LE/LEONT/ExtUtils-Config-0.008.tar.gz
provides:
ExtUtils::Config 0.008
requirements:
Data::Dumper 0
ExtUtils::MakeMaker 6.30
strict 0
warnings 0
ExtUtils-Helpers-0.026
pathname: L/LE/LEONT/ExtUtils-Helpers-0.026.tar.gz
provides:
ExtUtils::Helpers 0.026
ExtUtils::Helpers::Unix 0.026
ExtUtils::Helpers::VMS 0.026
ExtUtils::Helpers::Windows 0.026
requirements:
Carp 0
Exporter 5.57
ExtUtils::MakeMaker 0
File::Basename 0
File::Copy 0
File::Spec::Functions 0
Text::ParseWords 3.24
perl 5.006
strict 0
warnings 0
ExtUtils-InstallPaths-0.011
pathname: L/LE/LEONT/ExtUtils-InstallPaths-0.011.tar.gz
provides:
ExtUtils::InstallPaths 0.011
requirements:
Carp 0
ExtUtils::Config 0.002
ExtUtils::MakeMaker 0
File::Spec 0
perl 5.006
strict 0
warnings 0
File-Remove-1.52 File-Remove-1.52
pathname: A/AD/ADAMK/File-Remove-1.52.tar.gz pathname: A/AD/ADAMK/File-Remove-1.52.tar.gz
provides: provides:
...@@ -235,6 +274,34 @@ DISTRIBUTIONS ...@@ -235,6 +274,34 @@ DISTRIBUTIONS
requirements: requirements:
ExtUtils::MakeMaker 6.30 ExtUtils::MakeMaker 6.30
Locale::Maketext 1.17 Locale::Maketext 1.17
Minion-6.02
pathname: S/SR/SRI/Minion-6.02.tar.gz
provides:
Minion 6.02
Minion::Backend undef
Minion::Backend::Pg undef
Minion::Command::minion undef
Minion::Command::minion::job undef
Minion::Command::minion::worker undef
Minion::Job undef
Minion::Worker undef
Mojolicious::Plugin::Minion undef
requirements:
ExtUtils::MakeMaker 0
Mojolicious 6.0
perl 5.010001
Minion-Backend-SQLite-0.009
pathname: D/DB/DBOOK/Minion-Backend-SQLite-0.009.tar.gz
provides:
Minion::Backend::SQLite 0.009
requirements:
Minion 4.0
Module::Build::Tiny 0.034
Mojo::SQLite 1.002
Mojolicious 6.0
Sys::Hostname 0
Time::HiRes 0
perl 5.010001
Module-Build-0.4214 Module-Build-0.4214
pathname: L/LE/LEONT/Module-Build-0.4214.tar.gz pathname: L/LE/LEONT/Module-Build-0.4214.tar.gz
provides: provides:
...@@ -285,8 +352,55 @@ DISTRIBUTIONS ...@@ -285,8 +352,55 @@ DISTRIBUTIONS
Text::ParseWords 0 Text::ParseWords 0
perl 5.008000 perl 5.008000
version 0.87 version 0.87
Mojolicious-7.12 Module-Build-Tiny-0.039
pathname: S/SR/SRI/Mojolicious-7.12.tar.gz pathname: L/LE/LEONT/Module-Build-Tiny-0.039.tar.gz
provides:
Module::Build::Tiny 0.039
requirements:
CPAN::Meta 0
DynaLoader 0
Exporter 5.57
ExtUtils::CBuilder 0
ExtUtils::Config 0.003
ExtUtils::Helpers 0.020
ExtUtils::Install 0
ExtUtils::InstallPaths 0.002
ExtUtils::ParseXS 0
File::Basename 0
File::Find 0
File::Path 0
File::Spec::Functions 0
Getopt::Long 2.36
JSON::PP 2
Pod::Man 0
TAP::Harness::Env 0
perl 5.006
strict 0
warnings 0
Mojo-SQLite-1.003
pathname: D/DB/DBOOK/Mojo-SQLite-1.003.tar.gz
provides:
Mojo::SQLite 1.003
Mojo::SQLite::Database 1.003
Mojo::SQLite::Migrations 1.003
Mojo::SQLite::PubSub 1.003
Mojo::SQLite::Results 1.003
Mojo::SQLite::Transaction 1.003
requirements:
Carp 0
DBD::SQLite 1.50
DBI 1.627
File::Spec::Functions 0
File::Temp 0
Module::Build::Tiny 0.034
Mojolicious 6.14
Scalar::Util 0
URI 1.69
URI::db 0.15
URI::file 4.21
perl 5.010001
Mojolicious-7.13
pathname: S/SR/SRI/Mojolicious-7.13.tar.gz
provides: provides:
Mojo undef Mojo undef
Mojo::Asset undef Mojo::Asset undef
...@@ -350,7 +464,7 @@ DISTRIBUTIONS ...@@ -350,7 +464,7 @@ DISTRIBUTIONS
Mojo::UserAgent::Transactor undef Mojo::UserAgent::Transactor undef
Mojo::Util undef Mojo::Util undef
Mojo::WebSocket undef Mojo::WebSocket undef
Mojolicious 7.12 Mojolicious 7.13
Mojolicious::Command undef Mojolicious::Command undef
Mojolicious::Command::cgi undef Mojolicious::Command::cgi undef
Mojolicious::Command::cpanify undef Mojolicious::Command::cpanify undef
...@@ -632,3 +746,123 @@ DISTRIBUTIONS ...@@ -632,3 +746,123 @@ DISTRIBUTIONS
Test::Builder 0.32 Test::Builder 0.32
Test::More 0.96 Test::More 0.96
perl 5.006 perl 5.006
URI-1.71
pathname: E/ET/ETHER/URI-1.71.tar.gz
provides:
URI 1.71
URI::Escape 3.31
URI::Heuristic 4.20
URI::IRI 1.71
URI::QueryParam 1.71
URI::Split 1.71
URI::URL 5.04
URI::WithBase 2.20
URI::_foreign 1.71
URI::_generic 1.71
URI::_idna 1.71
URI::_ldap 1.71
URI::_login 1.71
URI::_punycode 1.71
URI::_query 1.71
URI::_segment 1.71
URI::_server 1.71
URI::_userpass 1.71
URI::data 1.71
URI::file 4.21
URI::file::Base 1.71
URI::file::FAT 1.71
URI::file::Mac 1.71
URI::file::OS2 1.71
URI::file::QNX 1.71
URI::file::Unix 1.71
URI::file::Win32 1.71
URI::ftp 1.71
URI::gopher 1.71
URI::http 1.71
URI::https 1.71
URI::ldap 1.71
URI::ldapi 1.71
URI::ldaps 1.71
URI::mailto 1.71
URI::mms 1.71
URI::news 1.71
URI::nntp 1.71
URI::pop 1.71
URI::rlogin 1.71
URI::rsync 1.71
URI::rtsp 1.71
URI::rtspu 1.71
URI::sftp 1.71
URI::sip 1.71
URI::sips 1.71
URI::snews 1.71
URI::ssh 1.71
URI::telnet 1.71
URI::tn3270 1.71
URI::urn 1.71
URI::urn::isbn undef
URI::urn::oid 1.71
requirements:
Exporter 5.57
ExtUtils::MakeMaker 0
MIME::Base64 2
Scalar::Util 0
parent 0
perl 5.008001
utf8 0
URI-Nested-0.10
pathname: D/DW/DWHEELER/URI-Nested-0.10.tar.gz
provides:
URI::Nested 0.10
requirements:
Module::Build 0.30
Test::More 0.88
URI 1.40
perl 5.008001
URI-db-0.17
pathname: D/DW/DWHEELER/URI-db-0.17.tar.gz
provides:
URI::cassandra 0.17
URI::couch 0.17
URI::couchdb 0.17
URI::cubrid 0.17
URI::db 0.17
URI::db2 0.17
URI::derby 0.17
URI::firebird 0.17
URI::hive 0.17
URI::impala 0.17
URI::informix 0.17
URI::ingres 0.17
URI::interbase 0.17
URI::ldapdb 0.17
URI::maria 0.17
URI::mariadb 0.17
URI::max 0.17
URI::maxdb 0.17
URI::monet 0.17
URI::monetdb 0.17
URI::mongo 0.17
URI::mongodb 0.17
URI::mssql 0.17
URI::mysql 0.17
URI::oracle 0.17
URI::pg 0.17
URI::pgsql 0.17
URI::pgxc 0.17
URI::postgres 0.17
URI::postgresql 0.17
URI::postgresxc 0.17
URI::sqlite 0.17
URI::sqlite3 0.17
URI::sqlserver 0.17
URI::sybase 0.17
URI::teradata 0.17
URI::unify 0.17
URI::vertica 0.17
requirements:
Module::Build 0.30
Test::More 0.88
URI 1.40
URI::Nested 0.10
perl 5.008001
...@@ -21,7 +21,11 @@ sub startup { ...@@ -21,7 +21,11 @@ sub startup {
secret => ['hfudsifdsih'], secret => ['hfudsifdsih'],
page_offset => 10, page_offset => 10,
theme => 'default', theme => 'default',
ban_min_strike => 3 ban_min_strike => 3,
minion => {
enabled => 0,
db_path => 'minion.db'
}
} }
}); });
...@@ -51,6 +55,11 @@ sub startup { ...@@ -51,6 +55,11 @@ sub startup {
# Piwik # Piwik
$self->plugin('Piwik'); $self->plugin('Piwik');
# Minion
if ($config->{minion}->{enabled} && $config->{minion}->{db_path}) {
$self->plugin('Minion' => { SQLite => 'sqlite:'.$config->{minion}->{db_path} });
}
# Schema updates # Schema updates
LstuModel->do('CREATE TABLE IF NOT EXISTS sessions (token TEXT PRIMARY KEY, until INTEGER)'); LstuModel->do('CREATE TABLE IF NOT EXISTS sessions (token TEXT PRIMARY KEY, until INTEGER)');
LstuModel->do('CREATE TABLE IF NOT EXISTS ban (ip TEXT PRIMARY KEY, until INTEGER, strike INTEGER)'); LstuModel->do('CREATE TABLE IF NOT EXISTS ban (ip TEXT PRIMARY KEY, until INTEGER, strike INTEGER)');
...@@ -203,11 +212,39 @@ sub startup { ...@@ -203,11 +212,39 @@ sub startup {
} }
} }
); );
$self->hook(after_static => sub { $self->hook(after_static => sub {
my $c = shift; my $c = shift;
$c->res->headers->cache_control('max-age=2592000, must-revalidate'); $c->res->headers->cache_control('max-age=2592000, must-revalidate');
}); });
# Minion
if ($config->{minion}->{enabled} && $config->{minion}->{db_path}) {
$self->app->minion->add_task(
increase_counter => sub {
my $job = shift;
my $short = shift;
my $url = shift;
my @urls = LstuModel::Lstu->select('WHERE short = ?', $short);
if (scalar(@urls)) {
$urls[0]->update(counter => $urls[0]->counter + 1);
}
my $piwik = $job->app->config('piwik');
if (defined($piwik) && $piwik->{idsite} && $piwik->{url}) {
$job->app->piwik_api(
'Track' => {
idSite => $piwik->{idsite},
action_url => $url,
url => $piwik->{url}
}
);
}
}
);
}
# For the first launch (after, this isn't really useful) # For the first launch (after, this isn't really useful)
$self->provisioning(); $self->provisioning();
......
...@@ -223,27 +223,30 @@ sub get { ...@@ -223,27 +223,30 @@ sub get {
); );
# Update counter # Update counter
$c->on(finish => sub { $c->on(finish => sub {
if (defined $c->cache->{$short}) { @urls = LstuModel::Lstu->select('WHERE short = ?', $short) if (defined $c->cache->{$short});
@urls = LstuModel::Lstu->select('WHERE short = ?', $short); $c->cache->{$short} = {
last_used => time,
url => $url
};
if ($c->config('minion')->{enabled} && $c->config('minion')->{db_path}) {
$c->app->minion->enqueue(increase_counter => [$short, $c->{url}]);
} else { } else {
$c->cache->{$short} = { my $counter = $urls[0]->counter + 1;
last_used => time, $urls[0]->update(counter => $counter);
url => $url
};
}
my $counter = $urls[0]->counter + 1;
$urls[0]->update(counter => $counter);
my $piwik = $c->config('piwik');
if (defined($piwik) && $piwik->{idsite} && $piwik->{url}) {
$c->piwik_api(
'Track' => {
idSite => $piwik->{idsite},
action_url => $c->{url},
url => $piwik->{url}
}
);
}
my $piwik = $c->config('piwik');
if (defined($piwik) && $piwik->{idsite} && $piwik->{url}) {
$c->piwik_api(
'Track' => {
idSite => $piwik->{idsite},
action_url => $c->{url},
url => $piwik->{url}
}
);
}
}
$c->clear_cache; $c->clear_cache;
}); });
} else { } else {
......
...@@ -86,5 +86,13 @@ ...@@ -86,5 +86,13 @@
#piwik => { #piwik => {
# url => 'http://piwik.example.com', # url => 'http://piwik.example.com',
# idsite => '1', # idsite => '1',
#} #},
# use Minion instead of directly increase counters
# need to launch a minion worker service if enabled
# optional, Minion is disabled by default
#minion => {
# enabled => 0,
# db_path => 'minion.db' # you can define it relative to lstu directory or set an absolute path
#},
}; };
...@@ -42,11 +42,15 @@ $t->get_ok($a.'i.json') ...@@ -42,11 +42,15 @@ $t->get_ok($a.'i.json')
->status_is(200) ->status_is(200)
->json_is({success => false, msg => 'The shortened URL '.$a.'i doesn\'t exist.'}); ->json_is({success => false, msg => 'The shortened URL '.$a.'i doesn\'t exist.'});
# Needed if we use Minion for increasing counters
sleep 2;
$t->get_ok('/stats.json') $t->get_ok('/stats.json')
->status_is(200) ->status_is(200)
->json_has('/0/created_at', '/0/counter', '/0/short', '/0/url') ->json_has('/0/created_at', '/0/counter', '/0/short', '/0/url')
->json_is('/0/url' => 'https://lstu.fr', '/0/short' => $a) ->json_is('/0/url' => 'https://lstu.fr', '/0/short' => $a)
->json_like('/0/created_at' => qr#\d+#, '/0/counter' => qr#\d+#); ->json_is('/0/counter' => 2)
->json_like('/0/created_at' => qr#\d+#);
my $b = $a; my $b = $a;
$b =~ s#http://127\.0\.0\.1:\d+/##; $b =~ s#http://127\.0\.0\.1:\d+/##;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment