Commit 8ab0bb2c by Luc Didry

Merge branch 'development'. Release version 0.02 \o/

parents 2f4a7676 0edc1074
......@@ -13,3 +13,4 @@
* Quentin Pagès (occitan translation)
* Yann Le Brech (htpasswd file support)
* Jéssica Da Cunha (portuguese translation)
* Ilker Kulgu (fix IE11 compatibility, Dutch translation)
Revision history for Lufi
0.02 2017-09-13
- Database abstraction layer
- PostgreSQL support
- IE 11 and Edge support
- Fix encoding error (#83)
- Htpassword authentication support
- Ability to add a password to a file
- Portuguese translation
- Catalan translation
- Dutch translation
- IE 11 support
- Deleting files from "My files" is now done with Ajax (#23)
- Allow bulk files deletion from "My files" (#24)
- Bugfixes
0.01 2017-01-09
- Upload files
- Download files
- Preview files in browser depending on mimetype (images and videos)
- LDAP authentication support
EXTRACTFILES=utilities/locales_files.txt
EXTRACTDIR=-D lib -D themes/default/templates
EN=themes/default/lib/Lufi/I18N/en.po
FR=themes/default/lib/Lufi/I18N/fr.po
IT=themes/default/lib/Lufi/I18N/it.po
OC=themes/default/lib/Lufi/I18N/oc.po
CA=themes/default/lib/Lufi/I18N/ca.po
PT=themes/default/lib/Lufi/I18N/pt.po
NL=themes/default/lib/Lufi/I18N/nl.po
XGETTEXT=carton exec local/bin/xgettext.pl
CARTON=carton exec
REAL_LUFI=script/application
LUFI=script/lufi
locales:
$(XGETTEXT) -f $(EXTRACTFILES) -o $(EN) 2>/dev/null
$(XGETTEXT) -f $(EXTRACTFILES) -o $(FR) 2>/dev/null
$(XGETTEXT) -f $(EXTRACTFILES) -o $(IT) 2>/dev/null
$(XGETTEXT) -f $(EXTRACTFILES) -o $(OC) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(EN) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(FR) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(IT) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(OC) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(CA) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(PT) 2>/dev/null
$(XGETTEXT) $(EXTRACTDIR) -o $(NL) 2>/dev/null
podcheck:
podchecker lib/Lufi/DB/File.pm lib/Lufi/DB/Slice.pm
test:
$(CARTON) $(REAL_LUFI) test
......
......@@ -4,6 +4,17 @@
Lufi means Let's Upload that FIle.
## Which browsers are compatible?
Lufi is tested and working on the following browsers / devices :
- Firefox
- Chrome
- Internet Explorer 11
- Microsoft Edge
- Safari
- iOS devices (ipad, iphone)
- Android devices (Galaxy tab, Galaxy S8)
## What does it do?
It stores files and allows you to download them.
......@@ -45,6 +56,10 @@ However please note that some metadata are sent unencrypted:
* its size
* its mimetype
## Client
There is the web interface, but you can use a CLI client too! Have a look at <https://framagit.org/luc/lufi-cli> or install it directly with `sudo npm install -g lufi-cli`.
## Internationalization
Lufi comes with English and French languages. It will choose the language to display from the browser's settings.
......
requires 'Mojolicious', '< 7.31';
requires 'Mojolicious', '>= 7.31';
requires 'ORLite';
requires 'Mojolicious::Plugin::DebugDumperHelper';
requires 'Mojolicious::Plugin::PgURLHelper';
requires 'Mojolicious::Plugin::I18N';
requires 'Mojolicious::Plugin::Mail';
requires 'Mojolicious::Plugin::Authentication';
requires 'Mojo::Pg';
requires 'EV';
requires 'Filesys::DiskUsage';
requires 'Switch';
......@@ -15,4 +17,5 @@ requires 'Filesys::DfPortable';
requires 'Switch';
requires 'Data::Entropy';
requires 'Net::LDAP';
requires 'Apache::Htpasswd';
\ No newline at end of file
requires 'Crypt::SaltedHash';
requires 'Apache::Htpasswd';
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi;
use Mojo::Base 'Mojolicious';
use LufiDB;
use Data::Entropy qw(entropy_source);
use Net::LDAP;
use Apache::Htpasswd;
......@@ -11,23 +9,26 @@ $ENV{MOJO_MAX_WEBSOCKET_SIZE} = 100485760; # 10 * 1024 * 1024 = 10MiB
# This method will run once at server start
sub startup {
my $self = shift;
my $entry;
my $config = $self->plugin('Config' => {
default => {
provisioning => 100,
provis_step => 5,
length => 10,
token_length => 32,
secrets => ['hfudsifdsih'],
default_delay => 0,
max_delay => 0,
mail => {
provisioning => 100,
provis_step => 5,
length => 10,
token_length => 32,
secrets => ['hfudsifdsih'],
default_delay => 0,
max_delay => 0,
mail => {
how => 'sendmail'
},
mail_sender => 'no-reply@lufi.io',
theme => 'default',
upload_dir => 'files',
session_duration => 3600,
mail_sender => 'no-reply@lufi.io',
theme => 'default',
upload_dir => 'files',
session_duration => 3600,
allow_pwd_on_files => 0,
dbtype => 'sqlite',
}
});
......@@ -83,34 +84,41 @@ sub startup {
my $mesg = $ldap->bind($c->config->{ldap}->{bind_user}.$c->config->{ldap}->{bind_dn},
password => $c->config->{ldap}->{bind_pwd}
);
$mesg->code && die $mesg->error;
$mesg = $ldap->search(
base => $c->config->{ldap}->{user_tree},
filter => "(&(uid=$username)".$c->config->{ldap}->{user_filter}.")"
);
if ($mesg->code) {
$c->app->log->error($mesg->error);
return undef;
}
# Now we know that the user exists
# we filtered out, but did we actually get a non-empty result?
$entry = $mesg->shift_entry;
if (!defined $entry) {
$c->app->log->info("[LDAP authentication failed] - User $username filtered out, IP: ".$c->ip);
return undef;
}
# Now we know that the user exists, and that he is authorized by the filter
$mesg = $ldap->bind('uid='.$username.$c->config->{ldap}->{bind_dn},
password => $password
);
if ($mesg->code) {
$c->app->log->info("[LDAP authentication failed] login: $username, IP: ".$c->ip);
$c->app->log->error("[LDAP authentication failed] ".$mesg->error);
return undef;
}
$c->app->log->info("[LDAP authentication successful] login: $username, IP: ".$c->ip);
} elsif (defined($c->config('htpasswd'))) {
my $htpasswd = new Apache::Htpasswd({passwdFile => $c->config->{htpasswd},
ReadOnly => 1}
ReadOnly => 1}
);
if (!$htpasswd->htCheckPassword($username, $password)) {
return undef;
......@@ -130,105 +138,7 @@ sub startup {
$self->secrets($self->config('secrets'));
# Helpers
$self->helper(
provisioning => sub {
my $c = shift;
# Create some short patterns for provisioning
if (LufiDB::Files->count('WHERE created_at IS NULL') < $c->config('provisioning')) {
for (my $i = 0; $i < $c->config('provis_step'); $i++) {
if (LufiDB->begin) {
my $short;
do {
$short= $c->shortener($c->config('length'));
} while (LufiDB::Files->count('WHERE short = ?', $short));
LufiDB::Files->create(
short => $short
);
LufiDB->commit;
}
}
}
}
);
$self->helper(
get_empty => sub {
my $c = shift;
my @records = LufiDB::Files->select('WHERE created_at IS NULL LIMIT 1');
return $records[0];
}
);
$self->helper(
shortener => sub {
my $c = shift;
my $length = shift;
my @chars = ('a'..'z','A'..'Z','0'..'9', '-', '_');
my $result = '';
foreach (1..$length) {
$result .= $chars[entropy_source->get_int(scalar(@chars))];
}
return $result;
}
);
$self->helper(
ip => sub {
my $c = shift;
my $proxy = $c->req->headers->header('X-Forwarded-For');
my $ip = ($proxy) ? $proxy : $c->tx->remote_address;
my $remote_port = (defined($c->req->headers->header('X-Remote-Port'))) ? $c->req->headers->header('X-Remote-Port') : $c->tx->remote_port;
return "$ip remote port:$remote_port";
}
);
$self->helper(
default_delay => sub {
my $c = shift;
return $c->config('default_delay') if ($c->config('default_delay') >= 0);
warn "default_delay set to a negative value. Default to 0.";
return 0;
}
);
$self->helper(
max_delay => sub {
my $c = shift;
return $c->config('max_delay') if ($c->config('max_delay') >= 0);
warn "max_delay set to a negative value. Default to 0.";
return 0;
}
);
$self->helper(
is_selected => sub {
my $c = shift;
my $num = shift;
return ($num == $c->max_delay) ? 'selected="selected"' : '' if ($c->max_delay && !$c->default_delay);
return ($num == $c->default_delay) ? 'selected="selected"' : '';
}
);
$self->helper(
stop_upload => sub {
my $c = shift;
if (-f 'stop-upload' || -f 'stop-upload.manual') {
return 1;
}
return 0;
}
);
$self->plugin('Lufi::Plugin::Helpers');
# Hooks
$self->hook(
after_dispatch => sub {
......@@ -277,9 +187,12 @@ sub startup {
if($c->authenticate($login, $pwd)) {
$c->redirect_to('index');
} else {
$c->stash(msg => $c->l('Please, check your credentials: unable to authenticate.'));
$c->render(template => 'login');
} elsif (defined $entry) {
$c->stash(msg => $c->l('Please, check your credentials: unable to authenticate.'));
$c->render(template => 'login');
} else {
$c->stash(msg => $c->l('Sorry mate, you are not authorised to use that service. Contact your sysadmin if you think there\'s a glitch in the matrix.'));
$c->render(template => 'login');
}
});
# Logout page
......@@ -297,6 +210,11 @@ sub startup {
shift->render(template => 'about');
})->name('about');
# Get instance stats
$r->get('/fullstats')
->to('Misc#fullstats')
->name('fullstats');
# Get a file
$r->get('/r/:short')->
to('Files#r')->
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron::cleanbdd;
use Mojo::Base 'Mojolicious::Command';
use LufiDB;
use Lufi::DB::File;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Delete IP addresses from database after configured delay.';
has usage => sub { shift->extract_usage };
......@@ -11,20 +10,24 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
keep_ip_during => 365,
}
});
my $separation = time() - $config->{keep_ip_during} * 86400;
LufiDB->do(
'UPDATE files SET created_by = NULL WHERE created_by IS NOT NULL AND created_at < ?',
{},
$separation
);
Lufi::DB::File->new(app => $c->app)->delete_creator_before($separation);
}
=encoding utf8
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Command::cron::cleanfiles;
use Mojo::Base 'Mojolicious::Command';
use LufiDB;
use Lufi::File;
use Lufi::DB::File;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Delete expired files.';
has usage => sub { shift->extract_usage };
......@@ -12,28 +10,39 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $time = time();
## Select only files expired since two days, to be sure that nobody is still downloading it
my @files = LufiDB::Files->select('WHERE deleted = 0 AND ((delete_at_day + 2) * 86400) < (? - created_at) AND delete_at_day != 0', $time);
for my $file (@files) {
my $f = Lufi::File->new(record => $file);
$file->delete;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
}
});
if (defined($config->{delete_no_longer_viewed_files}) && $config->{delete_no_longer_viewed_files} > 0) {
$time = time() - $config->{delete_no_longer_viewed_files} * 86400;
@files = LufiDB::Files->select('WHERE deleted = 0 AND last_access_at < ?', $time);
my $time = time();
for my $file (@files) {
my $f = Lufi::File->new(record => $file);
$file->delete;
my $ldfile = Lufi::DB::File->new(app => $c->app);
$ldfile->get_expired($time)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
if (defined($config->{delete_no_longer_viewed_files}) && $config->{delete_no_longer_viewed_files} > 0) {
$time = time() - $config->{delete_no_longer_viewed_files} * 86400;
my $ldfile = Lufi::DB::File->new(app => $c->app);
$ldfile->get_no_longer_viewed($time)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
}
}
......
package Lufi::Command::cron::watch;
use Mojo::Base 'Mojolicious::Command';
use Filesys::DiskUsage qw/du/;
use LufiDB;
use Lufi::File;
use Lufi::DB::File;
use Switch;
use FindBin qw($Bin);
use File::Spec qw(catfile);
has description => 'Watch the files directory and take action when over quota';
has usage => sub { shift->extract_usage };
......@@ -13,9 +11,17 @@ has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
my $cfile = Mojo::File->new($Bin, '..' , 'lufi.conf');
if (defined $ENV{MOJO_CONFIG}) {
$cfile = Mojo::File->new($ENV{MOJO_CONFIG});
unless (-e $cfile->to_abs) {
$cfile = Mojo::File->new($Bin, '..', $ENV{MOJO_CONFIG});
}
}
my $config = $c->app->plugin('Config', {
file => File::Spec->catfile($Bin, '..' ,'lufi.conf'),
file => $cfile,
default => {
dbtype => 'sqlite',
policy_when_full => 'warn'
}
});
......@@ -36,11 +42,14 @@ sub run {
}
case 'delete' {
say '[Lufi cron job watch] Older files are being deleted';
my $ldfile = Lufi::DB::File->new(app => $c->app);
do {
for my $file (LufiDB::Files->select('WHERE deleted = 0 ORDER BY created_at ASC LIMIT 50')) {
my $f = Lufi::File->new(record => $file);
$file->delete;
}
$ldfile->get_oldest_undeleted_files(50)->each(
sub {
my ($f, $num) = @_;
$f->delete;
}
);
} while (du(qw/files/) > $config->{max_total_size});
}
else {
......
......@@ -62,7 +62,7 @@ SEDOPTS=-e "s\@SOME DESCRIPTIVE TITLE\@Lufi language file\@" \\
-e "s\@CHARSET\@utf8\@" \\
-e "s\@the PACKAGE package\@the Lufi package\@" \\
-e '/^\\#\\. (/{N;/\\n\\#\\. (/{N;/\\n.*\\.\\.\\/default\\//{s/\\#\\..*\\n.*\\#\\./\\#. (/g}}}' \\
-e '/^\\#\\. (/{N;/\\n.*\\.\\.\\/default\\//{s/\\n/ /}}'
-e '/^\\#\\. (/{N;/\\n.*\\.\\.\\/default\\//{s/\\n/ /}}'
SEDOPTS2=-e '/^\\#.*\\.\\.\\/default\\//,+3d'
XGETTEXT=carton exec ../../local/bin/xgettext.pl
CARTON=carton exec
......
......@@ -6,10 +6,11 @@ use Mojo::JSON qw(decode_json);
sub render_mail {
my $c = shift;
my $links = (defined($c->param('links'))) ? decode_json($c->param('links')) : [];
$c->render(
template => 'mail',
links => decode_json($c->param('links'))
links => $links
);
}
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::Controller::Misc;
use Mojo::Base 'Mojolicious::Controller';
use Mojo::File;
use Lufi::DB::File;
sub fullstats {
my $c = shift;
my $stats = Lufi::DB::File->new(app => $c->app)->get_stats;
return $c->render(
json => {
files => $stats->{files},
deleted => $stats->{deleted},
empty => $stats->{empty},
timestamp => time,
}
);
}
1;
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::DB::File;
use Mojo::Base -base;
use Mojo::Collection;
has 'short';
has 'deleted' => 0;
has 'mediatype';
has 'filename';
has 'filesize';
has 'counter' => 0;
has 'delete_at_first_view' => 0;
has 'delete_at_day';
has 'created_at' => sub {
return time;
};
has 'created_by';
has 'last_access_at';
has 'mod_token';
has 'nbslices';
has 'complete' => 0;
has 'slices' => sub {
return Mojo::Collection->new();
};
has 'passwd';
has 'app';
=head1 NAME
Lufi::DB::File - DB abstraction layer for Lufi file
=head1 Contributing
When creating a new database accessor, make sure that it provides the following subroutines.
After that, modify this file and modify the C<new> subroutine to allow to use your accessor.
Have a look at Lufi::DB::File::SQLite's code: it's simple and may be more understandable that this doc.
=head1 Attributes
=over 1
=item B<short> : string
=item B<deleted> : boolean
=item B<mediatype> : string
=item B<filename> : string
=item B<filesize> : integer
=item B<counter> : integer
=item B<delete_at_first_view> : boolean
=item B<delete_at_day> : integer
=item B<created_at> : unix timestamp
=item B<created_by> : string
=item B<last_access_at> : unix timestamp
=item B<mod_token> : string
=item B<nbslices> : integer
=item B<complete> : boolean
=item B<slices> : Mojo::Collection of Lufi::DB::Slice
=item B<passwd> : string
=item B<app> : a Mojolicious object
=back
=head1 Sub routines
=head2 new
=over 1
=item B<Usage> : C<$c = Lufi::DB::File-E<gt>new(app =E<gt> $self);>
=item B<Arguments> : any of the attribute above
=item B<Purpose> : construct a new db accessor object. If the C<short> attribute is provided, it have to load the informations from the database.
=item B<Returns> : the db accessor object
=item B<Info> : the app argument is used by Lufi::DB::File to choose which db accessor will be used, you don't need to use it in new(), but you can use it to access helpers or configuration settings in the other subroutines
=back
=cut
sub new {
my $c = shift;
$c = $c->SUPER::new(@_);
if (ref($c) eq 'Lufi::DB::File') {
my $dbtype = $c->app->config('dbtype');
if ($dbtype eq 'sqlite') {
use Lufi::DB::File::SQLite;
$c = Lufi::DB::File::SQLite->new(@_);
} elsif ($dbtype eq 'postgresql') {
use Lufi::DB::File::Pg;
$c = Lufi::DB::File::Pg->new(@_);
}
}
return $c;
}
=head2 delete
=over 1
=item B<Usage> : C<$c-E<gt>delet>
=item B<Arguments> : none
=item B<Purpose> : delete the files of the slices and the directory containing those files, then update the object by setting the deleted attribute to 1 (true)
=item B<Returns> : the db accessor object
=back
=cut
sub delete {
my $c = shift;
$c->slices->each(sub {
my ($e, $num) = @_;
unlink $e->path;
});
rmdir Mojo::File->new($c->app->config('upload_dir'), $c->short);
$c->deleted(1);
$c->write;
return $c;
}
=head2 write
=over 1
=item B<Usage> : C<$c-E<gt>write>
=item B<Arguments> : none
=item B<Purpose> : create or update a record in the database, with the values of the object's attributes
=item B<Returns> : the db accessor object
=back
=head2 count_empty
=over 1
=item B<Usage> : C<$c-E<gt>count_empty>
=item B<Arguments> : none
=item B<Purpose> : count how many records have a null created_at column
=item B<Returns> : integer
=back
=head2 already_exists
=over 1
=item B<Usage> : C<$c-E<gt>already_exists($short)>
=item B<Arguments> : a string
=item B<Purpose> : check if the given string is already used as short attribute for a file
=item B<Returns> : 1 or 0
=back
=head2 get_empty
=over 1
=item B<Usage> : C<$c-E<gt>get_empty>
=item B<Arguments> : none
=item B<Purpose> : select an empty ready-to-use record from the database
=item B<Returns> : a db accessor object
=back
=head2 get_stats
=over 1
=item B<Usage> : C<$c-E<gt>get_stats>
=item B<Arguments> : none
=item B<Purpose> : get stats about how many empty files, deleted files and non-deleted files there is in the database
=item B<Returns> : a hash table reference containing three keys: files, deleted and empty
=back