Verified Commit d382a6bb authored by Luc Didry's avatar Luc Didry

Merge remote-tracking branch 'origin/development'

parents 439e2a0c f65a4c35
local/*
cover_db/*
safebrowsing_db/*
.tx/*
*.db
*.db-shm
......
Revision history for Perl application Lstu
0.19-0 2018-??-??
- Allow to use Google Safe Browsing database to check if the URLs are harmless
It uses a local copy of the GSB DB, so Google won't see the URLs to shorten
- safebrowsingcheck CLI to check all the URLs in the database against GSB DB.
usage: carton exec script/lstu safebrowsingcheck
0.18-1 2018-08-21
- Update url CLI help message
......
......@@ -47,3 +47,8 @@ feature 'mysql', 'MySQL support' => sub {
requires 'Minion::Backend::mysql';
requires 'Mojolicious::Plugin::PgURLHelper';
};
feature 'safebrowsing', 'Check URLs against Google safebrowsing database' => sub {
requires 'Net::Google::SafeBrowsing4';
requires 'Term::ProgressBar::Quiet';
};
This diff is collapsed.
......@@ -31,6 +31,7 @@ sub startup {
db_path => 'lstu.db',
max_redir => 2,
skip_spamhaus => 0,
safebrowsing_api_key => '',
memcached_servers => [],
x_frame_options => 'DENY',
x_content_type_options => 'nosniff',
......@@ -241,6 +242,13 @@ sub startup {
$self->provisioning();
});
if ($self->stash('gsb')) {
Mojo::IOLoop->recurring(86400 => sub {
my $loop = shift;
$self->gsb_update();
});
}
# Minion
if ($config->{minion}->{enabled}) {
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lstu::Command::safebrowsingcheck;
use Mojo::Base 'Mojolicious::Command';
use FindBin qw($Bin);
use File::Spec qw(catfile);
use Term::ProgressBar::Quiet;
has description => 'Checks all URLs in database against Google Safe Browsing database (local copy)';
has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my @args = @_;
my $cfile = Mojo::File->new($Bin, '..' , 'lstu.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => $cfile,
default => {
prefix => '/',
provisioning => 100,
provis_step => 5,
length => 8,
secret => ['hfudsifdsih'],
page_offset => 10,
theme => 'default',
ban_min_strike => 3,
ban_whitelist => [],
ban_blacklist => [],
minion => {
enabled => 0,
db_path => 'minion.db'
},
session_duration => 3600,
dbtype => 'sqlite',
db_path => 'lstu.db',
max_redir => 2,
skip_spamhaus => 0,
safebrowsing_api_key => '',
memcached_servers => [],
x_frame_options => 'DENY',
x_content_type_options => 'nosniff',
x_xss_protection => '1; mode=block',
log_creator_ip => 0,
}
});
$c->app->plugin('Lstu::Plugin::Helpers');
if ($c->app->gsb) {
my $urls = Lstu::DB::URL->new(app => $c->app)->get_all_urls;
my $progress = Term::ProgressBar::Quiet->new(
{ name => 'Scanning '.$urls->size.' URLs', count => $urls->size, ETA => 'linear' }
);
my @bad;
$urls->each(sub {
my ($e, $num) = @_;
$progress->update($num);
my @matches = $c->app->gsb->lookup(url => $e->{url});
if (@matches) {
push @bad, $e->{short};
}
});
say sprintf('All URLs (%d) have been scanned.', $urls->size);
say sprintf('%d bad URLs detected.', scalar(@bad));
say scalar($c->app->gsb->lookup(url => 'http://malware.testing.google.test/testing/malware/'));
say sprintf("If you want to delete the detected bad URLs, please do:\n carton exec script/lstu url --remove %s", join(' ', @bad)) if @bad;
} else {
say 'It seems that safebrowsing_api_key isn\'t set. Please, check your configuration';
}
}
=encoding utf8
=head1 NAME
Lstu::Command::safebrowsing - Checks all URLs in database against Google Safe Browsing database (local copy)
=head1 SYNOPSIS
Usage:
carton exec script/lstu safebrowsingcheck
=cut
1;
......@@ -63,7 +63,7 @@ sub run {
}
getopt \@args,
'i|info=s{1,}' => \my @info,
'info=s{1,}' => \my @info,
'r|remove=s{1,}' => \my @remove,
's|search=s' => \my $search,
'ip=s' => \my $ip,
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lstu::DB::Ban;
use Mojo::Base -base;
use Mojo::Collection 'c';
has 'ip';
has 'until';
......@@ -42,9 +43,9 @@ Have a look at Lstu::DB::Ban::SQLite's code: it's simple and may be more underst
=item B<Arguments> : any of the attribute above
=item B<Purpose> : construct a new db accessor object. If the C<ip> attribute is provided, it have to load the informations from the database
=item B<Purpose> : construct a new Lstu::DB::Ban object. If the C<ip> attribute is provided, it have to load the informations from the database
=item B<Returns> : the db accessor object
=item B<Returns> : the Lstu::DB::Ban object
=item B<Info> : the app argument is used by Lstu::DB::Ban to choose which db accessor will be used, you don't need to use it in new(), but you can use it to access helpers or configuration settings in the other subroutines
......@@ -104,7 +105,7 @@ sub is_whitelisted {
my $c = shift;
my $ip = $c->ip;
return scalar(grep(sub { $_ eq $ip }, @{$c->app->config('ban_whitelist')}));
return c(@{$c->app->config('ban_whitelist')})->grep(sub { $_ eq $ip })->size;
}
=head2 is_blacklisted
......@@ -127,7 +128,7 @@ sub is_blacklisted {
my $c = shift;
my $ip = $c->ip;
return scalar(grep(sub { $_ eq $ip }, @{$c->app->config('ban_blacklist')}));
return c(@{$c->app->config('ban_blacklist')})->grep(sub { $_ eq $ip })->size;
}
=head2 is_banned
......@@ -142,24 +143,51 @@ sub is_blacklisted {
eg: C<WHERE ip = ? AND until E<gt> ? AND strike E<gt>= ?', $c->ip, time, $argument>
=item B<Returns> : the db accessor object if the ip is banned, undef otherwise
=item B<Returns> : the Lstu::DB::Ban object if the ip is banned, undef otherwise
=item B<Info> : if the IP is whitelisted (see C<is_whitelisted> above), it must return undef
=back
=cut
sub is_banned {
my $c = shift;
my $ban_min_strike = shift;
return undef if $c->is_whitelisted;
if ($c->is_blacklisted) {
$c->until(time + 3600);
$c->strike(1 + $c->app->config('ban_min_strike'));
return $c;
}
my $h = $c->app->dbi->db->query('SELECT * FROM ban WHERE ip = ? AND until > ? AND strike >= ?', $c->ip, time, $ban_min_strike)->hashes;
if ($h->size) {
$c->until($h->first->{until});
$c->strike($h->first->{strike});
$c->record(1);
return $c;
} else {
return undef;
}
}
=head2 increment_ban_delay
=over 1
=item B<Usage> : C<$c-E<gt>increment_ban_delay(3600)>
=item B<Arguments> : an integer. This number is a penalty (in second) that will be added to the C<until> attribute of the db accessor object
=item B<Arguments> : an integer. This number is a penalty (in second) that will be added to the C<until> attribute of the Lstu::DB::Ban object
=item B<Purpose> : add penalty to the C<until> attribute of the db accessor object, increment the C<strike> attribute by one and write the db accessor object's attribute to the database.
=item B<Purpose> : add penalty to the C<until> attribute of the Lstu::DB::Ban object, increment the C<strike> attribute by one and write the Lstu::DB::Ban object's attribute to the database.
Update the database record if one already exists, create one otherwise.
=item B<Returns> : the db accessor object
=item B<Returns> : the Lstu::DB::Ban object
=back
......@@ -179,6 +207,14 @@ eg: C<WHERE until E<lt> ?, time>
=back
=cut
sub clear {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM ban WHERE until < ?', time);
}
=head2 delete_all
=over 1
......@@ -195,4 +231,39 @@ eg: C<WHERE until E<lt> ?, time>
=cut
sub delete_all {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM ban');
}
=head2 _slurp
=over 1
=item B<Usage> : C<$c-E<gt>_slurp>
=item B<Arguments> : none
=item B<Purpose> : put a database record's columns into the Lstu::DB::Ban object's attributes
=item B<Returns> : the Lstu::DB::Ban object
=back
=cut
sub _slurp {
my $c = shift;
my $h = $c->app->dbi->db->query('SELECT * FROM ban WHERE ip = ?', $c->ip)->hashes;
if ($h->size) {
$c->until($h->first->{until});
$c->strike($h->first->{strike});
$c->record(1);
}
return $c;
}
1;
......@@ -14,31 +14,6 @@ sub new {
return $c;
}
sub is_banned {
my $c = shift;
my $ban_min_strike = shift;
return undef if $c->is_whitelisted;
if ($c->is_blacklisted) {
$c->until(time + 3600);
$c->strike(1 + $c->app->config('ban_min_strike'));
return $c;
}
my $h = $c->app->mysql->db->query('SELECT * FROM ban WHERE ip = ? AND until > ? AND strike >= ?', $c->ip, time, $ban_min_strike)->hashes;
if ($h->size) {
$c->until($h->first->{until});
$c->strike($h->first->{strike});
$c->record(1);
return $c;
} else {
return undef;
}
}
sub increment_ban_delay {
my $c = shift;
my $penalty = shift;
......@@ -49,10 +24,10 @@ sub increment_ban_delay {
strike => 1
};
if ($c->record) {
$c->app->mysql->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ?', $until, $c->ip);
$h = $c->app->mysql->db->query('SELECT strike FROM ban WHERE ip = ?', $c->ip)->hashes->first;
$c->app->dbi->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ?', $until, $c->ip);
$h = $c->app->dbi->db->query('SELECT strike FROM ban WHERE ip = ?', $c->ip)->hashes->first;
} else {
$c->app->mysql->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->app->dbi->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->record(1);
}
......@@ -62,29 +37,4 @@ sub increment_ban_delay {
return $c;
}
sub clear {
my $c = shift;
$c->app->mysql->db->query('DELETE FROM ban WHERE until < ?', time);
}
sub delete_all {
my $c = shift;
$c->app->mysql->db->query('DELETE FROM ban');
}
sub _slurp {
my $c = shift;
my $h = $c->app->mysql->db->query('SELECT * FROM ban WHERE ip = ?', $c->ip)->hashes;
if ($h->size) {
$c->until($h->first->{until});
$c->strike($h->first->{strike});
$c->record(1);
}
return $c;
}
1;
......@@ -14,31 +14,6 @@ sub new {
return $c;
}
sub is_banned {
my $c = shift;
my $ban_min_strike = shift;
return undef if $c->is_whitelisted;
if ($c->is_blacklisted) {
$c->until(time + 3600);
$c->strike(1 + $c->app->config('ban_min_strike'));
return $c;
}
my $h = $c->app->pg->db->query('SELECT * FROM ban WHERE ip = ? AND until > ? AND strike >= ?', $c->ip, time, $ban_min_strike)->hashes;
if ($h->size) {
$c->until($h->first->{until});
$c->strike($h->first->{strike});
$c->record(1);
return $c;
} else {
return undef;
}
}
sub increment_ban_delay {
my $c = shift;
my $penalty = shift;
......@@ -49,9 +24,9 @@ sub increment_ban_delay {
strike => 1
};
if ($c->record) {
$h = $c->app->pg->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ? RETURNING strike', $until, $c->ip)->hashes->first;
$h = $c->app->dbi->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ? RETURNING strike', $until, $c->ip)->hashes->first;
} else {
$c->app->pg->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->app->dbi->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->record(1);
}
......@@ -61,29 +36,4 @@ sub increment_ban_delay {
return $c;
}
sub clear {
my $c = shift;
$c->app->pg->db->query('DELETE FROM ban WHERE until < ?', time);
}
sub delete_all {
my $c = shift;
$c->app->pg->db->query('DELETE FROM ban');
}
sub _slurp {
my $c = shift;
my $h = $c->app->pg->db->query('SELECT * FROM ban WHERE ip = ?', $c->ip)->hashes;
if ($h->size) {
$c->until($h->first->{until});
$c->strike($h->first->{strike});
$c->record(1);
}
return $c;
}
1;
......@@ -14,31 +14,6 @@ sub new {
return $c;
}
sub is_banned {
my $c = shift;
my $ban_min_strike = shift;
return undef if $c->is_whitelisted;
if ($c->is_blacklisted) {
$c->until(time + 3600);
$c->strike(1 + $c->app->config('ban_min_strike'));
return $c;
}
my $h = $c->app->sqlite->db->query('SELECT * FROM ban WHERE ip = ? AND until > ? AND strike >= ?', $c->ip, time, $ban_min_strike)->hashes;
if ($h->size) {
$c->until($h->first->{until});
$c->strike($h->first->{strike});
$c->record(1);
return $c;
} else {
return undef;
}
}
sub increment_ban_delay {
my $c = shift;
my $penalty = shift;
......@@ -49,10 +24,10 @@ sub increment_ban_delay {
strike => 1
};
if ($c->record) {
$c->app->sqlite->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ?', $until, $c->ip)->hashes->first;
$h = $c->app->sqlite->db->query('SELECT strike FROM ban WHERE ip = ?', $c->ip)->hashes->first;
$c->app->dbi->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ?', $until, $c->ip)->hashes->first;
$h = $c->app->dbi->db->query('SELECT strike FROM ban WHERE ip = ?', $c->ip)->hashes->first;
} else {
$c->app->sqlite->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->app->dbi->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->record(1);
}
......@@ -62,29 +37,4 @@ sub increment_ban_delay {
return $c;
}
sub clear {
my $c = shift;
$c->app->sqlite->db->query('DELETE FROM ban WHERE until < ?', time);
}
sub delete_all {
my $c = shift;
$c->app->sqlite->db->query('DELETE FROM ban');
}
sub _slurp {
my $c = shift;
my $h = $c->app->sqlite->db->query('SELECT * FROM ban WHERE ip = ?', $c->ip)->hashes;
if ($h->size) {
$c->until($h->first->{until});
$c->strike($h->first->{strike});
$c->record(1);
}
return $c;
}
1;
......@@ -39,9 +39,9 @@ Have a look at Lstu::DB::Session::SQLite's code: it's simple and may be more und
=item B<Arguments> : any of the attribute above
=item B<Purpose> : construct a new db accessor object. If the C<token> attribute is provided, it have to load the informations from the database
=item B<Purpose> : construct a new Lstu::DB::Session object. If the C<token> attribute is provided, it have to load the informations from the database
=item B<Returns> : the db accessor object
=item B<Returns> : the Lstu::DB::Session object
=item B<Info> : the app argument is used by Lstu::DB::Session to choose which db accessor will be used, you don't need to use it in new(), but you can use it to access helpers or configuration settings in the other subroutines
......@@ -97,10 +97,12 @@ sub to_hash {
=item B<Purpose> : delete the session record from the database
=item B<Returns> : the db accessor object
=item B<Returns> : the Lstu::DB::Session object
=back
=cut
=head2 write
=over 1
......@@ -111,10 +113,25 @@ sub to_hash {
=item B<Purpose> : create or update the object in the database
=item B<Returns> : the db accessor object
=item B<Returns> : the Lstu::DB::Session object
=back
=cut
sub write {
my $c = shift;
if ($c->record) {
$c->app->dbi->db->query('UPDATE sessions SET until = ? WHERE token = ?', $c->until, $c->token);
} else {
$c->app->dbi->db->query('INSERT INTO sessions (token, until) VALUES (?, ?)', $c->token, $c->until);
$c->record(1);
}
return $c;
}
=head2 clear
=over 1
......@@ -131,6 +148,14 @@ eg: C<WHERE until E<lt> ?, time>
=back
=cut
sub clear {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM sessions WHERE until < ?', time);
}
=head2 delete_all
=over 1
......@@ -147,4 +172,39 @@ eg: C<WHERE until E<lt> ?, time>
=cut
sub delete_all {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM sessions');
}
=head2 _slurp
=over 1
=item B<Usage> : C<$c-E<gt>_slurp>
=item B<Arguments> : none
=item B<Purpose> : put a database record's columns into the Lstu::DB::Session object's attributes
=item B<Returns> : the Lstu::DB::Session object
=back
=cut
sub _slurp {
my $c = shift;
my $h = $c->app->dbi->db->query('SELECT * FROM sessions WHERE token = ?', $c->token)->hashes;
if ($h->size) {
$c->token($h->first->{token});
$c->until($h->first->{until});
$c->record(1);
}
return $c;
}
1;
......@@ -17,8 +17,8 @@ sub new {
sub delete {
my $c = shift;
$c->app->mysql->db->query('DELETE FROM sessions WHERE token = ?', $c->token);
my $h = $c->app->mysql->db->query('SELECT * FROM sessions WHERE token = ?', $c->token)->hashes;
$c->app->dbi->db->query('DELETE FROM sessions WHERE token = ?', $c->token);
my $h = $c->app->dbi->db->query('SELECT * FROM sessions WHERE token = ?', $c->token)->hashes;
if ($h->size) {
# We found the session, it hasn't been deleted
return 0;
......@@ -29,42 +29,4 @@ sub delete {
}
}
sub write {
my $c = shift;
if ($c->record) {
$c->app->mysql->db->query('UPDATE sessions SET until = ? WHERE token = ?', $c->until, $c->token);
} else {
$c->app->mysql->db->query('INSERT INTO sessions (token, until) VALUES (?, ?)', $c->token, $c->until);
$c->record(1);
}
return $c;
}
sub clear {
my $c = shift;
$c->app->mysql->db->query('DELETE FROM sessions WHERE until < ?', time);
}
sub delete_all {
my $c = shift;
$c->app->mysql->db->query('DELETE FROM sessions');
}
sub _slurp {
my $c = shift;
my $h = $c->app->mysql->db->query('SELECT * FROM sessions WHERE token = ?', $c->token)->hashes;
if ($h->size) {
$c->token($h->first->{token});
$c->until($h->first->{until});
$c->record(1);
}
return $c;
}
1;
......@@ -17,7 +17,7 @@ sub new {
sub delete {
my $c = shift;
my $h = $c->app->pg->db->query('DELETE FROM sessions WHERE token = ? RETURNING *', $c->token)->hashes;
my $h = $c->app->dbi->db->query('DELETE FROM sessions WHERE token = ? RETURNING *', $c->token)->hashes;
if ($h->size) {
$c = Lstu::DB::Session->new(app => $c->app);
}
......@@ -25,42 +25,4 @@ sub delete {
return $h->size;
}
sub write {
my $c = shift;
if ($c->record) {
$c->app->pg->db->query('UPDATE sessions SET until = ? WHERE token = ?', $c->until, $c->token);
} else {
$c->app->pg->db->query('INSERT INTO sessions (token, until) VALUES (?, ?)', $c->token, $c->until);
$c->record(1);
}
return $c;
}
sub clear {
my $c = shift;
$c->app->pg->db->query('DELETE FROM sessions WHERE until < ?', time);
}