Commit d909b8ad authored by Luc Didry's avatar Luc Didry

Add DB abstraction layer

parent 5684cbe1
Revision history for Lufi
0.02 2017-?
- Database abstraction layer
- IE 11 and Edge support
- Fix encoding error (#83)
- Htpassword authentication support
- Ability to add a password to a file
0.01 2017-01-09
- Upload files
- Download files
- Preview files in browser depending on mimetype (images and videos)
- LDAP authentication support
......@@ -20,6 +20,9 @@ locales:
$(XGETTEXT) $(EXTRACTDIR) -o $(PT) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(NL) 2>/dev/null
podcheck:
podchecker lib/Lufi/DB/File.pm lib/Lufi/DB/Slice.pm
test:
$(CARTON) $(REAL_LUFI) test
......
requires 'Mojolicious', '>= 7.31';
requires 'ORLite';
requires 'Mojolicious::Plugin::DebugDumperHelper';
requires 'Mojolicious::Plugin::PgURLHelper';
requires 'Mojolicious::Plugin::I18N';
requires 'Mojolicious::Plugin::Mail';
requires 'Mojolicious::Plugin::Authentication';
requires 'Mojo::Pg';
requires 'EV';
requires 'Filesys::DiskUsage';
requires 'Switch';
......
......@@ -51,6 +51,26 @@ DISTRIBUTIONS
perl 5.006
strict 0
warnings 0
Class-Method-Modifiers-2.12
pathname: E/ET/ETHER/Class-Method-Modifiers-2.12.tar.gz
provides:
Class::Method::Modifiers 2.12
requirements:
B 0
Carp 0
Exporter 0
ExtUtils::MakeMaker 0
base 0
perl 5.006
strict 0
warnings 0
Clone-0.39
pathname: G/GA/GARU/Clone-0.39.tar.gz
provides:
Clone 0.39
requirements:
ExtUtils::MakeMaker 0
Test::More 0
Convert-ASN1-0.27
pathname: G/GB/GBARR/Convert-ASN1-0.27.tar.gz
provides:
......@@ -85,6 +105,17 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 6.30
Test::Fatal 0
Test::More 0
DBD-Pg-3.6.2
pathname: T/TU/TURNSTEP/DBD-Pg-3.6.2.tar.gz
provides:
Bundle::DBD::Pg v3.6.2
DBD::Pg v3.6.2
requirements:
DBI 1.614
ExtUtils::MakeMaker 6.11
Test::More 0.88
Time::HiRes 0
version 0
DBD-SQLite-1.54
pathname: I/IS/ISHIGAKI/DBD-SQLite-1.54.tar.gz
provides:
......@@ -241,6 +272,14 @@ DISTRIBUTIONS
perl 5.006
strict 0
warnings 0
Devel-GlobalDestruction-0.14
pathname: H/HA/HAARG/Devel-GlobalDestruction-0.14.tar.gz
provides:
Devel::GlobalDestruction 0.14
requirements:
ExtUtils::MakeMaker 0
Sub::Exporter::Progressive 0.001011
perl 5.006
Digest-HMAC-1.03
pathname: G/GA/GAAS/Digest-HMAC-1.03.tar.gz
provides:
......@@ -454,6 +493,14 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 0
HTTP::Headers 6
perl 5.008001
Hash-Merge-0.200
pathname: R/RE/REHSACK/Hash-Merge-0.200.tar.gz
provides:
Hash::Merge 0.200
requirements:
Clone 0
ExtUtils::MakeMaker 0
perl 5.008001
IO-HTML-1.001
pathname: C/CJ/CJM/IO-HTML-1.001.tar.gz
provides:
......@@ -571,6 +618,13 @@ DISTRIBUTIONS
File::Spec 0
List::Util 0
Test::More 0.47
MRO-Compat-0.13
pathname: H/HA/HAARG/MRO-Compat-0.13.tar.gz
provides:
MRO::Compat 0.13
requirements:
ExtUtils::MakeMaker 0
perl 5.006
MailTools-2.18
pathname: M/MA/MARKOV/MailTools-2.18.tar.gz
provides:
......@@ -654,6 +708,31 @@ DISTRIBUTIONS
Text::ParseWords 0
perl 5.006001
version 0.87
Module-Runtime-0.015
pathname: Z/ZE/ZEFRAM/Module-Runtime-0.015.tar.gz
provides:
Module::Runtime 0.015
requirements:
Module::Build 0
Test::More 0.41
perl 5.006
strict 0
warnings 0
Mojo-Pg-4.0
pathname: S/SR/SRI/Mojo-Pg-4.0.tar.gz
provides:
Mojo::Pg 4.0
Mojo::Pg::Database undef
Mojo::Pg::Migrations undef
Mojo::Pg::PubSub undef
Mojo::Pg::Results undef
Mojo::Pg::Transaction undef
requirements:
DBD::Pg 3.005001
ExtUtils::MakeMaker 0
Mojolicious 7.32
SQL::Abstract 1.81
perl 5.010001
Mojolicious-7.36
pathname: S/SR/SRI/Mojolicious-7.36.tar.gz
provides:
......@@ -813,6 +892,43 @@ DISTRIBUTIONS
Mojolicious 5
Test::More 0
perl 5.001001
Mojolicious-Plugin-PgURLHelper-0.03
pathname: L/LD/LDIDRY/Mojolicious-Plugin-PgURLHelper-0.03.tar.gz
provides:
Mojolicious::Plugin::PgURLHelper 0.03
requirements:
ExtUtils::MakeMaker 0
Mojolicious 7.23
Moo-2.003002
pathname: H/HA/HAARG/Moo-2.003002.tar.gz
provides:
Method::Generate::Accessor undef
Method::Generate::BuildAll undef
Method::Generate::Constructor undef
Method::Generate::DemolishAll undef
Moo 2.003002
Moo::HandleMoose undef
Moo::HandleMoose::FakeConstructor undef
Moo::HandleMoose::FakeMetaClass undef
Moo::HandleMoose::_TypeMap undef
Moo::Object undef
Moo::Role 2.003002
Moo::_Utils undef
Moo::_mro undef
Moo::_strictures undef
Moo::sification undef
oo undef
requirements:
Class::Method::Modifiers 1.1
Devel::GlobalDestruction 0.11
Exporter 5.57
ExtUtils::MakeMaker 0
Module::Runtime 0.014
Role::Tiny 2.000004
Scalar::Util 0
Sub::Defer 2.003001
Sub::Quote 2.003001
perl 5.006
Mozilla-CA-20160104
pathname: A/AB/ABH/Mozilla-CA-20160104.tar.gz
provides:
......@@ -909,6 +1025,45 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 6.30
File::Spec 0
strict 0
Role-Tiny-2.000005
pathname: H/HA/HAARG/Role-Tiny-2.000005.tar.gz
provides:
Role::Tiny 2.000005
Role::Tiny::With 2.000005
requirements:
Exporter 5.57
perl 5.006
SQL-Abstract-1.84
pathname: I/IL/ILMARI/SQL-Abstract-1.84.tar.gz
provides:
SQL::Abstract 1.84
SQL::Abstract::Test undef
SQL::Abstract::Tree undef
requirements:
Exporter 5.57
ExtUtils::MakeMaker 0
Hash::Merge 0.12
List::Util 0
MRO::Compat 0.12
Moo 2.000001
Scalar::Util 0
Sub::Quote 2.000001
Text::Balanced 2.00
Sub-Exporter-Progressive-0.001013
pathname: F/FR/FREW/Sub-Exporter-Progressive-0.001013.tar.gz
provides:
Sub::Exporter::Progressive 0.001013
requirements:
ExtUtils::MakeMaker 0
Sub-Quote-2.004000
pathname: H/HA/HAARG/Sub-Quote-2.004000.tar.gz
provides:
Sub::Defer 2.004000
Sub::Quote 2.004000
requirements:
ExtUtils::MakeMaker 0
Scalar::Util 0
perl 5.006
Sub-Uplevel-0.2800
pathname: D/DA/DAGOLDEN/Sub-Uplevel-0.2800.tar.gz
provides:
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi;
use Mojo::Base 'Mojolicious';
use LufiDB;
use Net::LDAP;
use Apache::Htpasswd;
......@@ -29,6 +28,7 @@ sub startup {
upload_dir => 'files',
session_duration => 3600,
allow_pwd_on_files => 0,
dbtype => 'sqlite',
}
});
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron::cleanbdd;
use Mojo::Base 'Mojolicious::Command';
use LufiDB;
use Lufi::DB::File;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Delete IP addresses from database after configured delay.';
has usage => sub { shift->extract_usage };
......@@ -11,20 +10,24 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
keep_ip_during => 365,
}
});
my $separation = time() - $config->{keep_ip_during} * 86400;
LufiDB->do(
'UPDATE files SET created_by = NULL WHERE created_by IS NOT NULL AND created_at < ?',
{},
$separation
);
Lufi::DB::File->new(app => $c->app)->delete_creator_before($separation);
}
=encoding utf8
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron::cleanfiles;
use Mojo::Base 'Mojolicious::Command';
use LufiDB;
use Lufi::File;
use Lufi::DB::File;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Delete expired files.';
has usage => sub { shift->extract_usage };
......@@ -12,28 +10,39 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $time = time();
## Select only files expired since two days, to be sure that nobody is still downloading it
my @files = LufiDB::Files->select('WHERE deleted = 0 AND ((delete_at_day + 2) * 86400) < (? - created_at) AND delete_at_day != 0', $time);
for my $file (@files) {
my $f = Lufi::File->new(record => $file);
$file->delete;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
}
});
if (defined($config->{delete_no_longer_viewed_files}) && $config->{delete_no_longer_viewed_files} > 0) {
$time = time() - $config->{delete_no_longer_viewed_files} * 86400;
@files = LufiDB::Files->select('WHERE deleted = 0 AND last_access_at < ?', $time);
my $time = time();
for my $file (@files) {
my $f = Lufi::File->new(record => $file);
$file->delete;
my $ldfile = Lufi::DB::File->new(app => $c->app);
$ldfile->get_expired($time)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
if (defined($config->{delete_no_longer_viewed_files}) && $config->{delete_no_longer_viewed_files} > 0) {
$time = time() - $config->{delete_no_longer_viewed_files} * 86400;
my $ldfile = Lufi::DB::File->new(app => $c->app);
$ldfile->get_no_longer_viewed($time)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
}
}
......
package Lufi::Command::cron::watch;
use Mojo::Base 'Mojolicious::Command';
use Filesys::DiskUsage qw/du/;
use LufiDB;
use Lufi::File;
use Lufi::DB::File;
use Switch;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Watch the files directory and take action when over quota';
has usage => sub { shift->extract_usage };
......@@ -13,9 +11,17 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
policy_when_full => 'warn'
}
});
......@@ -36,11 +42,14 @@ sub run {
}
case 'delete' {
say '[Lufi cron job watch] Older files are being deleted';
my $ldfile = Lufi::DB::File->new(app => $c->app);
do {
for my $file (LufiDB::Files->select('WHERE deleted = 0 ORDER BY created_at ASC LIMIT 50')) {
my $f = Lufi::File->new(record => $file);
$file->delete;
}
$ldfile->get_oldest_undeleted_files(50)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
} while (du(qw/files/) > $config->{max_total_size});
}
else {
......
This diff is collapsed.
......@@ -2,22 +2,18 @@
package Lufi::Controller::Misc;
use Mojo::Base 'Mojolicious::Controller';
use Mojo::File;
use LufiDB;
use Lufi::File;
use Lufi::Slice;
use Lufi::DB::File;
sub fullstats {
my $c = shift;
my $files = LufiDB::Files->count('WHERE created_at IS NOT null AND deleted = 0');
my $deleted = LufiDB::Files->count('WHERE created_at IS NOT null AND deleted = 1');
my $empty = LufiDB::Files->count('WHERE created_at IS null');
my $stats = Lufi::DB::File->new(app => $c->app)->get_stats;
return $c->render(
json => {
files => $files,
deleted => $deleted,
empty => $empty,
files => $stats->{files},
deleted => $stats->{deleted},
empty => $stats->{empty},
timestamp => time,
}
);
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::DB::File;
use Mojo::Base -base;
use Mojo::Collection;
has 'short';
has 'deleted' => 0;
has 'mediatype';
has 'filename';
has 'filesize';
has 'counter' => 0;
has 'delete_at_first_view' => 0;
has 'delete_at_day';
has 'created_at' => sub {
return time;
};
has 'created_by';
has 'last_access_at';
has 'mod_token';
has 'nbslices';
has 'complete' => 0;
has 'slices' => sub {
return Mojo::Collection->new();
};
has 'passwd';
has 'app';
=head1 NAME
Lufi::DB::File - DB abstraction layer for Lufi file
=head1 Contributing
When creating a new database accessor, make sure that it provides the following subroutines.
After that, modify this file and modify the C<new> subroutine to allow to use your accessor.
Have a look at Lufi::DB::File::SQLite's code: it's simple and may be more understandable that this doc.
=head1 Attributes
=over 1
=item B<short> : string
=item B<deleted> : boolean
=item B<mediatype> : string
=item B<filename> : string
=item B<filesize> : integer
=item B<counter> : integer
=item B<delete_at_first_view> : boolean
=item B<delete_at_day> : integer
=item B<created_at> : unix timestamp
=item B<created_by> : string
=item B<last_access_at> : unix timestamp
=item B<mod_token> : string
=item B<nbslices> : integer
=item B<complete> : boolean
=item B<slices> : Mojo::Collection of Lufi::DB::Slice
=item B<passwd> : string
=item B<app> : a Mojolicious object
=back
=head1 Sub routines
=head2 new
=over 1
=item B<Usage> : C<$c = Lufi::DB::File-E<gt>new(app =E<gt> $self);>
=item B<Arguments> : any of the attribute above
=item B<Purpose> : construct a new db accessor object. If the C<short> attribute is provided, it have to load the informations from the database.
=item B<Returns> : the db accessor object
=item B<Info> : the app argument is used by Lufi::DB::File to choose which db accessor will be used, you don't need to use it in new(), but you can use it to access helpers or configuration settings in the other subroutines
=back
=cut
sub new {
my $c = shift;
$c = $c->SUPER::new(@_);
if (ref($c) eq 'Lufi::DB::File') {
my $dbtype = $c->app->config('dbtype');
if ($dbtype eq 'sqlite') {
use Lufi::DB::File::SQLite;
$c = Lufi::DB::File::SQLite->new(@_);
#} elsif ($dbtype eq 'postgresql') {
#use Lufi::DB::File::Pg;
#$c = Lufi::DB::File::Pg->new(@_);
}
}
return $c;
}
=head2 delete
=over 1
=item B<Usage> : C<$c-E<gt>delet>
=item B<Arguments> : none
=item B<Purpose> : delete the files of the slices and the directory containing those files, then update the object by setting the deleted attribute to 1 (true)
=item B<Returns> : the db accessor object
=back
=cut
sub delete {
my $c = shift;
$c->slices->each(sub {
my ($e, $num) = @_;
unlink $e->path;
});
rmdir Mojo::File->new($c->app->config('upload_dir'), $c->short);
$c->deleted(1);
$c->write;
return $c;
}
=head2 write
=over 1
=item B<Usage> : C<$c-E<gt>write>
=item B<Arguments> : none
=item B<Purpose> : create or update a record in the database, with the values of the object's attributes
=item B<Returns> : the db accessor object
=back
=head2 count_empty
=over 1
=item B<Usage> : C<$c-E<gt>count_empty>
=item B<Arguments> : none
=item B<Purpose> : count how many records have a null created_at column
=item B<Returns> : integer
=back
=head2 already_exists
=over 1
=item B<Usage> : C<$c-E<gt>already_exists($short)>
=item B<Arguments> : a string
=item B<Purpose> : check if the given string is already used as short attribute for a file
=item B<Returns> : 1 or 0
=back
=head2 get_empty
=over 1
=item B<Usage> : C<$c-E<gt>get_empty>
=item B<Arguments> : none
=item B<Purpose> : select an empty ready-to-use record from the database
=item B<Returns> : a db accessor object
=back
=head2 get_stats
=over 1
=item B<Usage> : C<$c-E<gt>get_stats>
=item B<Arguments> : none
=item B<Purpose> : get stats about how many empty files, deleted files and non-deleted files there is in the database
=item B<Returns> : a hash table reference containing three keys: files, deleted and empty
=back
=head2 from_short
=over 1
=item B<Usage> : C<$c-E<gt>from_short($short)>
=item B<Arguments> : string
=item B<Purpose> : find a file in the database from its short attribute
=item B<Returns> : a db accessor object
=back
=head2 get_oldest_undeleted_files
=over 1
=item B<Usage> : C<$c-E<gt>get_oldest_undeleted_files($num)>
=item B<Arguments> : integer
=item B<Purpose> : get the X oldest non-deleted files
=item B<Returns> : a Mojo::Collection of Lufi::DB::File objects
=back
=head2 get_expired
=over 1
=item B<Usage> : C<$c-E<gt>get_expired($time)>
=item B<Arguments> : unix timestamp
=item B<Purpose> : get the non-deleted files that are expired at the given timestamp minus 2 days
=item B<Returns> : a Mojo::Collection of Lufi::DB::File objects
=back
=head2 get_no_longer_viewed
=over 1
=item B<Usage> : C<$c-E<gt>get_no_longer_viewed($time)>
=item B<Arguments> : unix timestamp
=item B<Purpose> : get the files that have not been viewed after the given timestamp
=item B<Returns> : a Mojo::Collection of Lufi::DB::File objects
=back
=head2 delete_creator_before
=over 1
=item B<Usage> : C<$c-E<gt>delete_creator_before($time)>
=item B<Arguments> : unix timestamp
=item B<Purpose> : empty the created_by column for files created before the given timestamp
=item B<Returns> : nothing
=back
=cut
1;
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::DB::File::SQLite;
use Mojo::Base 'Lufi::DB::File';
use Mojo::File;
use Mojo::Collection 'c';
use Lufi::DB::SQLite;
use Lufi::DB::Slice;
has 'record';
sub new {
my $c = shift;
$c = $c->SUPER::new(@_);
$c = $c->_slurp if defined $c->record;
return $c;