Verified Commit f61748ed authored by Luc Didry's avatar Luc Didry

Allow to use Google Safe Browsing database to check if the URLs are harmless

It uses a local copy of the GSB DB, so Google won't see the URLs to shorten
+ safebrowsingcheck CLI to check all the URLs in the database against GSB DB.
  usage: carton exec script/lstu safebrowsingcheck

This commit is dedicated to Schoumi, who is supporting me on Tipeee.
Many thanks :-)
parent 439e2a0c
Pipeline #73123 passed with stages
in 7 minutes and 49 seconds
local/*
cover_db/*
safebrowsing_db/*
.tx/*
*.db
*.db-shm
......
Revision history for Perl application Lstu
0.19-0 2018-??-??
- Allow to use Google Safe Browsing database to check if the URLs are harmless
It uses a local copy of the GSB DB, so Google won't see the URLs to shorten
- safebrowsingcheck CLI to check all the URLs in the database against GSB DB.
usage: carton exec script/lstu safebrowsingcheck
0.18-1 2018-08-21
- Update url CLI help message
......
......@@ -47,3 +47,8 @@ feature 'mysql', 'MySQL support' => sub {
requires 'Minion::Backend::mysql';
requires 'Mojolicious::Plugin::PgURLHelper';
};
feature 'safebrowsing', 'Check URLs against Google safebrowsing database' => sub {
requires 'Net::Google::SafeBrowsing4';
requires 'Term::ProgressBar::Quiet';
};
This diff is collapsed.
......@@ -31,6 +31,7 @@ sub startup {
db_path => 'lstu.db',
max_redir => 2,
skip_spamhaus => 0,
safebrowsing_api_key => '',
memcached_servers => [],
x_frame_options => 'DENY',
x_content_type_options => 'nosniff',
......@@ -241,6 +242,13 @@ sub startup {
$self->provisioning();
});
if ($self->stash('gsb')) {
Mojo::IOLoop->recurring(86400 => sub {
my $loop = shift;
$self->gsb_update();
});
}
# Minion
if ($config->{minion}->{enabled}) {
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lstu::Command::safebrowsingcheck;
use Mojo::Base 'Mojolicious::Command';
use FindBin qw($Bin);
use File::Spec qw(catfile);
use Term::ProgressBar::Quiet;
has description => 'Checks all URLs in database against Google Safe Browsing database (local copy)';
has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my @args = @_;
my $cfile = Mojo::File->new($Bin, '..' , 'lstu.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => $cfile,
default => {
prefix => '/',
provisioning => 100,
provis_step => 5,
length => 8,
secret => ['hfudsifdsih'],
page_offset => 10,
theme => 'default',
ban_min_strike => 3,
ban_whitelist => [],
ban_blacklist => [],
minion => {
enabled => 0,
db_path => 'minion.db'
},
session_duration => 3600,
dbtype => 'sqlite',
db_path => 'lstu.db',
max_redir => 2,
skip_spamhaus => 0,
safebrowsing_api_key => '',
memcached_servers => [],
x_frame_options => 'DENY',
x_content_type_options => 'nosniff',
x_xss_protection => '1; mode=block',
log_creator_ip => 0,
}
});
$c->app->plugin('Lstu::Plugin::Helpers');
if ($c->app->gsb) {
my $urls = Lstu::DB::URL->new(app => $c->app)->get_all_urls;
my $progress = Term::ProgressBar::Quiet->new(
{ name => 'Scanning '.$urls->size.' URLs', count => $urls->size, ETA => 'linear' }
);
my @bad;
$urls->each(sub {
my ($e, $num) = @_;
$progress->update($num);
my @matches = $c->app->gsb->lookup(url => $e->{url});
if (@matches) {
push @bad, $e->{short};
}
});
say sprintf('All URLs (%d) have been scanned.', $urls->size);
say sprintf('%d bad URLs detected.', scalar(@bad));
say scalar($c->app->gsb->lookup(url => 'http://malware.testing.google.test/testing/malware/'));
say sprintf("If you want to delete the detected bad URLs, please do:\n carton exec script/lstu url --remove %s", join(' ', @bad)) if @bad;
} else {
say 'It seems that safebrowsing_api_key isn\'t set. Please, check your configuration';
}
}
=encoding utf8
=head1 NAME
Lstu::Command::safebrowsing - Checks all URLs in database against Google Safe Browsing database (local copy)
=head1 SYNOPSIS
Usage:
carton exec script/lstu safebrowsingcheck
=cut
1;
......@@ -276,6 +276,21 @@ eg: C<COUNT(short) WHERE url IS NOT NULL>
=back
=head2 get_all_urls
=over 1
=item B<Usage> : C<$c-E<gt>get_all_urls()>
=item B<Arguments> : none
=item B<Purpose> : return all non-empty records
=item B<Returns> : a Mojo::Collection containing hashes of all non-empty records
=back
=cut
1;
......@@ -127,6 +127,12 @@ sub search_creator {
$c->app->mysql->db->select('lstu', undef, { created_by => $s })->hashes;
}
sub get_all_urls {
my $c = shift;
$c->app->mysql->db->select('lstu', undef, { url => { '!=', undef } })->hashes;
}
sub _slurp {
my $c = shift;
......
......@@ -125,6 +125,12 @@ sub search_creator {
$c->app->pg->db->select('lstu', undef, { created_by => $s })->hashes;
}
sub get_all_urls {
my $c = shift;
$c->app->pg->db->select('lstu', undef, { url => { '!=', undef } })->hashes;
}
sub _slurp {
my $c = shift;
......
......@@ -127,6 +127,12 @@ sub search_creator {
$c->app->sqlite->db->select('lstu', undef, { created_by => $s })->hashes;
}
sub get_all_urls {
my $c = shift;
$c->app->sqlite->db->select('lstu', undef, { url => { '!=', undef } })->hashes;
}
sub _slurp {
my $c = shift;
......
......@@ -5,6 +5,7 @@ use Net::Abuse::Utils::Spamhaus qw(check_fqdn);
use Lstu::DB::URL;
use Lstu::DB::Ban;
use Lstu::DB::Session;
use FindBin qw($Bin);
sub register {
my ($self, $app) = @_;
......@@ -59,6 +60,8 @@ sub register {
$app->helper(shortener => \&_shortener);
$app->helper(is_spam => \&_is_spam);
$app->helper(cleaning => \&_cleaning);
$app->helper(gsb => \&_gsb);
$app->helper(gsb_update => \&_gsb_update);
}
sub _sqlite {
......@@ -169,6 +172,12 @@ sub _is_spam {
msg => $c->l('The URL host or one of its redirection(s) (%1) is blacklisted at Spamhaus. I refuse to shorten it.', $url->host)
}
} else {
if ($c->config('safebrowsing_api_key') && scalar($c->gsb->lookup(url => $url->to_string))) {
return {
is_spam => 1,
msg => $c->l('The URL or one of its redirection(s) (%1) is blacklisted in Google Safe Browsing database. I refuse to shorten it.', $url)
}
}
my $res = $c->ua->get($url)->res;
if (defined($res->code) && $res->code >= 300 && $res->code < 400) {
my $new_url = Mojo::URL->new($res->headers->location);
......@@ -196,4 +205,40 @@ sub _cleaning {
Lstu::DB::Ban->new(app => $c)->clear();
}
sub _gsb {
my $c = shift;
# Google safebrowsing (if configured)
if ($c->config('safebrowsing_api_key')) {
use Net::Google::SafeBrowsing4;
use Net::Google::SafeBrowsing4::Storage::File;
my $force_update = (!-e Mojo::File->new($Bin, '..' , 'safebrowsing_db'));
my $storage = Net::Google::SafeBrowsing4::Storage::File->new(path => Mojo::File->new($Bin, '..' , 'safebrowsing_db'));
state $gsb = Net::Google::SafeBrowsing4->new(
key => $c->config('safebrowsing_api_key'),
storage => $storage,
);
$c->gsb_update($force_update);
return $gsb;
} else {
return undef;
}
}
sub _gsb_update {
my $c = shift;
my $force_update = shift;
return $c->gsb->update() if $force_update;
my $update = Mojo::File->new($Bin, '..' , 'safebrowsing_db')->to_string;
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($update);
$c->gsb->update() if ($mtime < time - 86400);
}
1;
......@@ -40,6 +40,7 @@ sub startup {
db_path => 'lstu.db',
max_redir => 2,
skip_spamhaus => 0,
safebrowsing_api_key => '',
memcached_servers => [],
x_frame_options => 'DENY',
x_content_type_options => 'nosniff',
......
......@@ -218,6 +218,14 @@
# optional, default is 0
#skip_spamhaus => 0,
# put your Google API key to enable Google safebrowsing check
# This will allow Lstu to download the Google safebrowsing database and use a local copy to check the URLs.
# Google does not get the URLs that are checked.
# Instructions to get a key: https://developers.google.com/safe-browsing/v4/get-started
# TL;DR: https://console.developers.google.com/projectselector/apis/library
# optional, no default
#safebrowsing_api_key => '',
# array of memcached servers to cache URL in order to accelerate responses to often-viewed URL.
# If set to [], the cache is disabled
# optional, default is []
......
......@@ -211,17 +211,22 @@ msgid "Statistics"
msgstr ""
#. ($url->host)
#: lib/Lstu/Plugin/Helpers.pm:169
#: lib/Lstu/Plugin/Helpers.pm:172
msgid "The URL host or one of its redirection(s) (%1) is blacklisted at Spamhaus. I refuse to shorten it."
msgstr ""
#. ($url)
#: lib/Lstu/Plugin/Helpers.pm:178
msgid "The URL or one of its redirection(s) (%1) is blacklisted in Google Safe Browsing database. I refuse to shorten it."
msgstr ""
#. ($c->config('max_redir')
#: lib/Lstu/Plugin/Helpers.pm:185
#: lib/Lstu/Plugin/Helpers.pm:194
msgid "The URL redirects %1 times or most. It's most likely a dangerous URL (spam, phishing, etc.). I refuse to shorten it."
msgstr ""
#. ($url->host)
#: lib/Lstu/Plugin/Helpers.pm:161
#: lib/Lstu/Plugin/Helpers.pm:164
msgid "The URL you want to shorten comes from a domain (%1) that is blacklisted on this server (usually because of spammers that use this domain)."
msgstr ""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment