Commit 40deab60 authored by Luc Didry's avatar Luc Didry
Browse files

Add cron tasks + stop-upload file check

parent 077b922f
......@@ -4,3 +4,4 @@ lufi.conf
lufi.db
*.swp
script/hypnotoad.pid
stop-upload
......@@ -10,3 +10,5 @@ requires 'Locale::Maketext';
requires 'Locale::Maketext::Extract';
requires 'Email::Valid';
requires 'Number::Bytes::Human';
requires 'Filesys::DfPortable';
requires 'Switch';
......@@ -183,6 +183,12 @@ DISTRIBUTIONS
File::Spec 3.29
Test::More 0.42
perl 5.00503
Filesys-DfPortable-0.85
pathname: I/IG/IGUTHRIE/Filesys-DfPortable-0.85.tar.gz
provides:
Filesys::DfPortable 0.85
requirements:
ExtUtils::MakeMaker 0
Filesys-DiskUsage-0.08
pathname: S/SZ/SZABGAB/Filesys-DiskUsage-0.08.tar.gz
provides:
......
......@@ -135,6 +135,16 @@ sub startup {
}
);
$self->helper(
stop_upload => sub {
my $c = shift;
if (-f 'stop-upload' || -f 'stop-upload.manual') {
return 1;
}
return 0;
}
);
# Hooks
$self->hook(
after_dispatch => sub {
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron;
use Mojo::Base 'Mojolicious::Commands';
has description => 'Execute tasks.';
has hint => <<EOF;
See 'script/lufi cron help TASK' for more information on a specific task.
EOF
has message => sub { shift->extract_usage . "\nCron tasks:\n" };
has namespaces => sub { ['Lufi::Command::cron'] };
sub help { shift->run(@_) }
1;
=encoding utf8
=head1 NAME
Lufi::Command::cron - Cron commands
=head1 SYNOPSIS
Usage: script/lufi cron TASK [OPTIONS]
=cut
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron::cleanbdd;
use Mojo::Base 'Mojolicious::Command';
use LufiDB;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Delete IP addresses from database after configured delay.';
has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
default => {
keep_ip_during => 365,
}
});
my $separation = time() - $config->{keep_ip_during} * 86400;
LufiDB->do(
'UPDATE files SET created_by = NULL WHERE created_by IS NOT NULL AND created_at < ?',
{},
$separation
);
}
=encoding utf8
=head1 NAME
Lufi::Command::cron::cleanbdd - Delete IP addresses from database after configured delay
=head1 SYNOPSIS
Usage: script/lufi cron cleanbdd
=cut
1;
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron::cleanfiles;
use Mojo::Base 'Mojolicious::Command';
use LufiDB;
use Lufi::File;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Delete expired files.';
has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $time = time();
## Select only files expired since two days, to be sure that nobody is still downloading it
my @files = LufiDB::Files->select('WHERE deleted = 0 AND ((delete_at_day + 2) * 86400) < (? - created_at) AND delete_at_day != 0', $time);
for my $file (@files) {
my $f = Lufi::File->new(record => $file);
$file->delete;
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
});
if (defined($config->{delete_no_longer_viewed_files}) && $config->{delete_no_longer_viewed_files} > 0) {
$time = time() - $config->{delete_no_longer_viewed_files} * 86400;
@files = LufiDB::Files->select('WHERE deleted = 0 AND last_access_at < ?', $time);
for my $file (@files) {
my $f = Lufi::File->new(record => $file);
$file->delete;
}
}
}
=encoding utf8
=head1 NAME
Lufi::Command::cron::cleanfiles - Delete expired files
=head1 SYNOPSIS
Usage: script/lufi cron cleanfiles
=cut
1;
package Lufi::Command::cron::watch;
use Mojo::Base 'Mojolicious::Command';
use Filesys::DiskUsage qw/du/;
use LufiDB;
use Lufi::File;
use Switch;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Watch the files directory and take action when over quota';
has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
default => {
policy_when_full => 'warn'
}
});
if (defined($config->{max_total_size})) {
my $total = du(qw/files/);
if ($total > $config->{max_total_size}) {
say "[Lufi cron job watch] Files directory is over quota ($total > ".$config->{max_total_size}.")";
switch ($config->{policy_when_full}) {
case 'warn' {
say "[Lufi cron job watch] Please, delete some files or increase quota (".$config->{max_total_size}.")";
}
case 'stop-upload' {
open (my $fh, '>', 'stop-upload') or die ("Couldn't open stop-upload: $!");
close($fh);
say '[Lufi cron job watch] Uploads are stopped. Delete some images and the stop-upload file to reallow uploads.';
}
case 'delete' {
say '[Lufi cron job watch] Older files are being deleted';
do {
for my $file (LufiDB::Files->select('WHERE deleted = 0 ORDER BY created_at ASC LIMIT 50')) {
my $f = Lufi::File->new(record => $file);
$file->delete;
}
} while (du(qw/files/) > $config->{max_total_size});
}
else {
say '[Lufi cron job watch] Unrecognized policy_when_full option: '.$config->{policy_when_full}.'. Aborting.';
}
}
} else {
unlink 'stop-upload' if (-f 'stop-upload');
}
} else {
say "[Lufi cron job watch] No max_total_size found in the configuration file. Aborting.";
}
}
=encoding utf8
=head1 NAME
Lufi::Command::cron::watch - Watch the files directory and take action when over quota
=head1 SYNOPSIS
Usage: script/lufi cron watch
=cut
1;
......@@ -8,6 +8,7 @@ use Lufi::File;
use Lufi::Slice;
use File::Spec::Functions;
use Number::Bytes::Human qw(format_bytes);
use Filesys::DfPortable;
sub upload {
my $c = shift;
......@@ -24,16 +25,27 @@ sub upload {
$c->debug('Got message');
my $over_size = 0;
my $stop = 0;
# Check if stop_upload file is present
if ($c->stop_upload) {
$stop = 1;
$c->send(sprintf('{"success": false, "msg":"'.$c->l('Sorry, uploading is disabled.').'", "sent_delay": %d, "i": %d}', $json->{delay}, $json->{i}));
}
# Check against max_size
if (defined $c->config('max_file_size')) {
elsif (defined $c->config('max_file_size')) {
if ($json->{size} > $c->config('max_file_size')) {
$over_size = 1;
$stop = 1;
$c->send(sprintf('{"success": false, "msg":"'.$c->l('Your file is too big: %1 (maximum size allowed: %2)', format_bytes($json->{size}), format_bytes($c->config('max_file_size'))).'", "sent_delay": %d, "i": %d}', $json->{delay}, $json->{i}));
}
}
# Check that we have enough space (multiplying by 2 since it's encrypted, it takes more place that the original file)
elsif (($json->{size} * 2) >= dfportable('files')->{bavail}) {
$stop = 1;
$c->send(sprintf('{"success": false, "msg":"'.$c->l('No enough space available on the server for this file (size: %1).', format_bytes($json->{size})).'", "sent_delay": %d, "i": %d}', $json->{delay}, $json->{i}));
}
unless ($over_size) {
unless ($stop) {
my $f;
if (defined($json->{id})) {
my @records = LufiDB::Files->select('WHERE short = ?', $json->{id});
......@@ -114,16 +126,28 @@ sub download {
# Do we have a file?
if (scalar @records) {
my $record = $records[0];
# Is the file fully uploaded?
if ($records[0]->deleted) {
if ($record->deleted
|| (
$record->delete_at_day != 0
&& (
($record->created_at + $record->delete_at_day * 86400) < time()
)
)
) {
unless ($record->deleted) {
my $f = Lufi::File->new(record => $record);
$f->delete;
}
$c->on(
message => sub {
my ($ws, $json) = @_;
$c->send('{"success": false, "msg": "'.$c->l('Error: the file existed but has been deleted.').'"}');
}
);
} elsif ($records[0]->complete) {
my $f = Lufi::File->new(record => $records[0]);
} elsif ($record->complete) {
my $f = Lufi::File->new(record => $record);
$c->on(
message => sub {
......
......@@ -7,6 +7,8 @@ use File::Spec qw(catfile);
sub startup {
my $self = shift;
push @{$self->commands->namespaces}, 'Lufi::Command';
my $config = $self->plugin('Config' =>
{
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
......
......@@ -100,4 +100,29 @@
# remember that it has to be in a directory writable by Lutim user
# optional, default is lufi.db
#db_path => 'lufi.db',
#########################
# Lufi cron jobs settings
#########################
# number of days senders' IP addresses are kept in database
# after that delay, they will be deleted from database (used with script/lufi cron cleanbdd)
# optional, default is 365
#keep_ip_during => 365,
# max size of the files directory, in octets
# used by script/lufi cron watch to trigger an action
# optional, no default
#max_total_size => 10*1024*1024*1024,
# default action when files directory is over max_total_size (used with script/lufi cron watch)
# valid values are 'warn', 'stop-upload' and 'delete'
# please, see readme
# optional, default is 'warn'
#policy_when_full => 'warn',
# images which are not viewed since delete_no_longer_viewed_files days will be deleted by the cron cleanfiles task
# if delete_no_longer_viewed_files is not set, the no longer viewed files will NOT be deleted
# optional, no default
#delete_no_longer_viewed_files => 90
};
......@@ -89,9 +89,12 @@ function destroyBlock(el) {
el.parentNode.parentNode.remove();
var a = document.getElementsByClassName('link-input');
var l = document.getElementById('results').querySelector('li');
if (a.length === 0) {
document.getElementById('misc').innerHTML = '';
document.getElementById('results').style.display = 'none';
if (l === null) {
document.getElementById('results').style.display = 'none';
}
} else {
updateMailLink();
}
......
......@@ -13,6 +13,13 @@
</div>
</div>
% }
% if (stop_upload) {
<div class="col-sm-12 col-xs-12">
<div class="alert alert-danger">
<strong><%= l('Sorry, the uploading is currently disabled. Please try again later.') %></strong>
</div>
</div>
% } else {
<div class="col-sm-1 col-xs-12">
<a href="<%= url_for('/delays') %>" title="<%= l('Important: more information on delays') %>"><span class="lead icon icon-help-circled"></span></a>
</div>
......@@ -82,3 +89,4 @@ var i18n = {
%= javascript '/js/sjcl.js'
%= javascript '/js/moment-with-locales.min.js'
%= javascript '/js/lufi-up.js'
% }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment