Commit d909b8ad authored by Luc Didry's avatar Luc Didry
Browse files

Add DB abstraction layer

parent 5684cbe1
Revision history for Lufi
0.02 2017-?
- Database abstraction layer
- IE 11 and Edge support
- Fix encoding error (#83)
- Htpassword authentication support
- Ability to add a password to a file
0.01 2017-01-09
- Upload files
- Download files
- Preview files in browser depending on mimetype (images and videos)
- LDAP authentication support
......@@ -20,6 +20,9 @@ locales:
$(XGETTEXT) $(EXTRACTDIR) -o $(PT) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(NL) 2>/dev/null
podcheck:
podchecker lib/Lufi/DB/File.pm lib/Lufi/DB/Slice.pm
test:
$(CARTON) $(REAL_LUFI) test
......
requires 'Mojolicious', '>= 7.31';
requires 'ORLite';
requires 'Mojolicious::Plugin::DebugDumperHelper';
requires 'Mojolicious::Plugin::PgURLHelper';
requires 'Mojolicious::Plugin::I18N';
requires 'Mojolicious::Plugin::Mail';
requires 'Mojolicious::Plugin::Authentication';
requires 'Mojo::Pg';
requires 'EV';
requires 'Filesys::DiskUsage';
requires 'Switch';
......
......@@ -51,6 +51,26 @@ DISTRIBUTIONS
perl 5.006
strict 0
warnings 0
Class-Method-Modifiers-2.12
pathname: E/ET/ETHER/Class-Method-Modifiers-2.12.tar.gz
provides:
Class::Method::Modifiers 2.12
requirements:
B 0
Carp 0
Exporter 0
ExtUtils::MakeMaker 0
base 0
perl 5.006
strict 0
warnings 0
Clone-0.39
pathname: G/GA/GARU/Clone-0.39.tar.gz
provides:
Clone 0.39
requirements:
ExtUtils::MakeMaker 0
Test::More 0
Convert-ASN1-0.27
pathname: G/GB/GBARR/Convert-ASN1-0.27.tar.gz
provides:
......@@ -85,6 +105,17 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 6.30
Test::Fatal 0
Test::More 0
DBD-Pg-3.6.2
pathname: T/TU/TURNSTEP/DBD-Pg-3.6.2.tar.gz
provides:
Bundle::DBD::Pg v3.6.2
DBD::Pg v3.6.2
requirements:
DBI 1.614
ExtUtils::MakeMaker 6.11
Test::More 0.88
Time::HiRes 0
version 0
DBD-SQLite-1.54
pathname: I/IS/ISHIGAKI/DBD-SQLite-1.54.tar.gz
provides:
......@@ -241,6 +272,14 @@ DISTRIBUTIONS
perl 5.006
strict 0
warnings 0
Devel-GlobalDestruction-0.14
pathname: H/HA/HAARG/Devel-GlobalDestruction-0.14.tar.gz
provides:
Devel::GlobalDestruction 0.14
requirements:
ExtUtils::MakeMaker 0
Sub::Exporter::Progressive 0.001011
perl 5.006
Digest-HMAC-1.03
pathname: G/GA/GAAS/Digest-HMAC-1.03.tar.gz
provides:
......@@ -454,6 +493,14 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 0
HTTP::Headers 6
perl 5.008001
Hash-Merge-0.200
pathname: R/RE/REHSACK/Hash-Merge-0.200.tar.gz
provides:
Hash::Merge 0.200
requirements:
Clone 0
ExtUtils::MakeMaker 0
perl 5.008001
IO-HTML-1.001
pathname: C/CJ/CJM/IO-HTML-1.001.tar.gz
provides:
......@@ -571,6 +618,13 @@ DISTRIBUTIONS
File::Spec 0
List::Util 0
Test::More 0.47
MRO-Compat-0.13
pathname: H/HA/HAARG/MRO-Compat-0.13.tar.gz
provides:
MRO::Compat 0.13
requirements:
ExtUtils::MakeMaker 0
perl 5.006
MailTools-2.18
pathname: M/MA/MARKOV/MailTools-2.18.tar.gz
provides:
......@@ -654,6 +708,31 @@ DISTRIBUTIONS
Text::ParseWords 0
perl 5.006001
version 0.87
Module-Runtime-0.015
pathname: Z/ZE/ZEFRAM/Module-Runtime-0.015.tar.gz
provides:
Module::Runtime 0.015
requirements:
Module::Build 0
Test::More 0.41
perl 5.006
strict 0
warnings 0
Mojo-Pg-4.0
pathname: S/SR/SRI/Mojo-Pg-4.0.tar.gz
provides:
Mojo::Pg 4.0
Mojo::Pg::Database undef
Mojo::Pg::Migrations undef
Mojo::Pg::PubSub undef
Mojo::Pg::Results undef
Mojo::Pg::Transaction undef
requirements:
DBD::Pg 3.005001
ExtUtils::MakeMaker 0
Mojolicious 7.32
SQL::Abstract 1.81
perl 5.010001
Mojolicious-7.36
pathname: S/SR/SRI/Mojolicious-7.36.tar.gz
provides:
......@@ -813,6 +892,43 @@ DISTRIBUTIONS
Mojolicious 5
Test::More 0
perl 5.001001
Mojolicious-Plugin-PgURLHelper-0.03
pathname: L/LD/LDIDRY/Mojolicious-Plugin-PgURLHelper-0.03.tar.gz
provides:
Mojolicious::Plugin::PgURLHelper 0.03
requirements:
ExtUtils::MakeMaker 0
Mojolicious 7.23
Moo-2.003002
pathname: H/HA/HAARG/Moo-2.003002.tar.gz
provides:
Method::Generate::Accessor undef
Method::Generate::BuildAll undef
Method::Generate::Constructor undef
Method::Generate::DemolishAll undef
Moo 2.003002
Moo::HandleMoose undef
Moo::HandleMoose::FakeConstructor undef
Moo::HandleMoose::FakeMetaClass undef
Moo::HandleMoose::_TypeMap undef
Moo::Object undef
Moo::Role 2.003002
Moo::_Utils undef
Moo::_mro undef
Moo::_strictures undef
Moo::sification undef
oo undef
requirements:
Class::Method::Modifiers 1.1
Devel::GlobalDestruction 0.11
Exporter 5.57
ExtUtils::MakeMaker 0
Module::Runtime 0.014
Role::Tiny 2.000004
Scalar::Util 0
Sub::Defer 2.003001
Sub::Quote 2.003001
perl 5.006
Mozilla-CA-20160104
pathname: A/AB/ABH/Mozilla-CA-20160104.tar.gz
provides:
......@@ -909,6 +1025,45 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 6.30
File::Spec 0
strict 0
Role-Tiny-2.000005
pathname: H/HA/HAARG/Role-Tiny-2.000005.tar.gz
provides:
Role::Tiny 2.000005
Role::Tiny::With 2.000005
requirements:
Exporter 5.57
perl 5.006
SQL-Abstract-1.84
pathname: I/IL/ILMARI/SQL-Abstract-1.84.tar.gz
provides:
SQL::Abstract 1.84
SQL::Abstract::Test undef
SQL::Abstract::Tree undef
requirements:
Exporter 5.57
ExtUtils::MakeMaker 0
Hash::Merge 0.12
List::Util 0
MRO::Compat 0.12
Moo 2.000001
Scalar::Util 0
Sub::Quote 2.000001
Text::Balanced 2.00
Sub-Exporter-Progressive-0.001013
pathname: F/FR/FREW/Sub-Exporter-Progressive-0.001013.tar.gz
provides:
Sub::Exporter::Progressive 0.001013
requirements:
ExtUtils::MakeMaker 0
Sub-Quote-2.004000
pathname: H/HA/HAARG/Sub-Quote-2.004000.tar.gz
provides:
Sub::Defer 2.004000
Sub::Quote 2.004000
requirements:
ExtUtils::MakeMaker 0
Scalar::Util 0
perl 5.006
Sub-Uplevel-0.2800
pathname: D/DA/DAGOLDEN/Sub-Uplevel-0.2800.tar.gz
provides:
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi;
use Mojo::Base 'Mojolicious';
use LufiDB;
use Net::LDAP;
use Apache::Htpasswd;
......@@ -29,6 +28,7 @@ sub startup {
upload_dir => 'files',
session_duration => 3600,
allow_pwd_on_files => 0,
dbtype => 'sqlite',
}
});
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron::cleanbdd;
use Mojo::Base 'Mojolicious::Command';
use LufiDB;
use Lufi::DB::File;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Delete IP addresses from database after configured delay.';
has usage => sub { shift->extract_usage };
......@@ -11,20 +10,24 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
keep_ip_during => 365,
}
});
my $separation = time() - $config->{keep_ip_during} * 86400;
LufiDB->do(
'UPDATE files SET created_by = NULL WHERE created_by IS NOT NULL AND created_at < ?',
{},
$separation
);
Lufi::DB::File->new(app => $c->app)->delete_creator_before($separation);
}
=encoding utf8
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron::cleanfiles;
use Mojo::Base 'Mojolicious::Command';
use LufiDB;
use Lufi::File;
use Lufi::DB::File;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Delete expired files.';
has usage => sub { shift->extract_usage };
......@@ -12,28 +10,39 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $time = time();
## Select only files expired since two days, to be sure that nobody is still downloading it
my @files = LufiDB::Files->select('WHERE deleted = 0 AND ((delete_at_day + 2) * 86400) < (? - created_at) AND delete_at_day != 0', $time);
for my $file (@files) {
my $f = Lufi::File->new(record => $file);
$file->delete;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
}
});
if (defined($config->{delete_no_longer_viewed_files}) && $config->{delete_no_longer_viewed_files} > 0) {
$time = time() - $config->{delete_no_longer_viewed_files} * 86400;
@files = LufiDB::Files->select('WHERE deleted = 0 AND last_access_at < ?', $time);
my $time = time();
for my $file (@files) {
my $f = Lufi::File->new(record => $file);
$file->delete;
my $ldfile = Lufi::DB::File->new(app => $c->app);
$ldfile->get_expired($time)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
if (defined($config->{delete_no_longer_viewed_files}) && $config->{delete_no_longer_viewed_files} > 0) {
$time = time() - $config->{delete_no_longer_viewed_files} * 86400;
my $ldfile = Lufi::DB::File->new(app => $c->app);
$ldfile->get_no_longer_viewed($time)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
}
}
......
package Lufi::Command::cron::watch;
use Mojo::Base 'Mojolicious::Command';
use Filesys::DiskUsage qw/du/;
use LufiDB;
use Lufi::File;
use Lufi::DB::File;
use Switch;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Watch the files directory and take action when over quota';
has usage => sub { shift->extract_usage };
......@@ -13,9 +11,17 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
policy_when_full => 'warn'
}
});
......@@ -36,11 +42,14 @@ sub run {
}
case 'delete' {
say '[Lufi cron job watch] Older files are being deleted';
my $ldfile = Lufi::DB::File->new(app => $c->app);
do {
for my $file (LufiDB::Files->select('WHERE deleted = 0 ORDER BY created_at ASC LIMIT 50')) {
my $f = Lufi::File->new(record => $file);
$file->delete;
}
$ldfile->get_oldest_undeleted_files(50)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
} while (du(qw/files/) > $config->{max_total_size});
}
else {
......
......@@ -4,9 +4,8 @@ use Mojo::Base 'Mojolicious::Controller';
use Mojo::JSON qw(encode_json decode_json to_json true false);
use Mojo::Util qw(encode decode);
use Mojo::File;
use LufiDB;
use Lufi::File;
use Lufi::Slice;
use Lufi::DB::File;
use Lufi::DB::Slice;
use File::Spec::Functions;
use Number::Bytes::Human qw(format_bytes);
use Filesys::DfPortable;
......@@ -38,47 +37,46 @@ sub upload {
# Check if stop_upload file is present
if ($c->stop_upload) {
$stop = 1;
$c->send(encode_json(
$c->send(decode('UTF-8', encode_json(
{
success => false,
msg => $c->l('Sorry, uploading is disabled.'),
sent_delay => $json->{delay},
i => $json->{i}
}
));
)));
}
# Check against max_size
elsif (defined $c->config('max_file_size')) {
if ($json->{size} > $c->config('max_file_size')) {
$stop = 1;
$c->send(encode_json(
$c->send(decode('UTF-8', encode_json(
{
success => false,
msg => $c->l('Your file is too big: %1 (maximum size allowed: %2)', format_bytes($json->{size}), format_bytes($c->config('max_file_size'))),
sent_delay => $json->{delay},
i => $json->{i}
}
));
)));
}
}
# Check that we have enough space (multiplying by 2 since it's encrypted, it takes more place that the original file)
elsif ($json->{part} == 0 && ($json->{size} * 2) >= dfportable($c->config('upload_dir'))->{bavail}) {
$stop = 1;
$c->send(encode_json(
$c->send(decode('UTF-8', encode_json(
{
success => false,
msg => $c->l('No enough space available on the server for this file (size: %1).', format_bytes($json->{size})),
sent_delay => $json->{delay},
i => $json->{i}
}
));
)));
}
unless ($stop) {
my $f;
if (defined($json->{id})) {
my @records = LufiDB::Files->select('WHERE short = ?', $json->{id});
$f = Lufi::File->new(record => $records[0]) if scalar @records;
$f = Lufi::DB::File->new(app => $c->app)->from_short($json->{id});
} else {
my $delay;
unless (defined $json->{delay}) {
......@@ -113,19 +111,17 @@ sub upload {
if (defined($c->config('ldap')) || defined($c->config('htpasswd'))) {
$creator = 'User: '.$c->current_user.', IP: '.$creator;
}
$f = Lufi::File->new(
record => $c->get_empty,
created_by => $creator,
delete_at_first_view => ($json->{del_at_first_view}) ? 1 : 0,
delete_at_day => $delay,
mediatype => $json->{type},
filename => $json->{name},
filesize => $json->{size},
nbslices => $json->{total},
mod_token => $c->shortener($c->config('token_length')),
passwd => $salted_pwd
);
$f->write;
$f = Lufi::DB::File->new(app => $c->app)->get_empty()
->created_by($creator)
->delete_at_first_view(($json->{del_at_first_view}) ? 1 : 0)
->delete_at_day($delay)
->mediatype($json->{type})
->filename($json->{name})
->filesize($json->{size})
->nbslices($json->{total})
->mod_token($c->shortener($c->config('token_length')))
->passwd($salted_pwd)
->write;
}
# This check is just in case we didn't succeed to find a corresponding record
......@@ -140,7 +136,8 @@ sub upload {
# Create slice file
my $file = catfile($dir, $json->{part}.'.part');
my $s = Lufi::Slice->new(
my $s = Lufi::DB::Slice->new(
app => $c->app,
short => $f->short,
j => $json->{part},
path => $file
......@@ -176,14 +173,14 @@ sub upload {
}
));
} else {
$ws->send(encode_json(
$ws->send(decode('UTF-8', encode_json(
{
success => false,
msg => $c->l('The server was unable to find the file record to add your file part to. Please, contact the administrator.'),
sent_delay => $json->{delay},
i => $json->{i}
}
));
)));
}
}
}
......@@ -203,37 +200,35 @@ sub download {
$c->inactivity_timeout(300000);
$c->app->log->debug('Client connected');
my @records = LufiDB::Files->select('WHERE short = ?', $short);
my $ldfile = Lufi::DB::File->new(app => $c->app)->from_short($short);
# Do we have a file?
if (scalar @records) {
my $record = $records[0];
if (defined $ldfile) {
# Is the file fully uploaded?
if ($record->deleted
if ($ldfile->deleted
|| (
$record->delete_at_day != 0
$ldfile->delete_at_day != 0
&& (
($record->created_at + $record->delete_at_day * 86400) < time()
($ldfile->created_at + $ldfile->delete_at_day * 86400) < time()
)
)
) {
unless ($record->deleted) {
my $f = Lufi::File->new(record => $record);
$f->delete;
unless ($ldfile->deleted) {
$ldfile->delete;
}
$c->on(
message => sub {
my ($ws, $json) = @_;
$c->send(encode_json(
$c->send(decode('UTF-8', encode_json(
{
success => false,
msg => $c->l('Error: the file existed but was deleted.')
}
));
)));
}
);
} elsif ($record->complete) {
my $f = Lufi::File->new(record => $record);
} elsif ($ldfile->complete) {
my $f = $ldfile;
$c->on(
message => sub {
......@@ -273,7 +268,11 @@ sub download {
}
}
} else {
$c->send(encode_json({msg => $c->l('Your password is not valid. Please refresh the page to retry.')}));
$c->send(decode('UTF-8', encode_json(
{
msg => $c->l('Your password is not valid. Please refresh the page to retry.')
}
)));
}
}
);
......@@ -286,22 +285,22 @@ sub download {
$c->on(
message => sub {
my ($ws, $json) = @_;
$c->send(encode_json(
$c->send(decode('UTF-8', encode_json(
{
success => false,
msg => $c->l('Error: the file has not been sent entirely.')
}
));
)));
}
);
}
} else {
$c->send(encode_json(
$c->send(decode('UTF-8', encode_json(
{