Verified Commit 5b43702d authored by Luc Didry's avatar Luc Didry

πŸ”€ β€” Merge branch 'development' of framagit.org:fiat-tux/hat-softwares/lufi

parents 1b3502a1 d562b313
......@@ -58,6 +58,8 @@ variables:
services:
- name: rroemhild/test-openldap
alias: rroemhild-test-openldap
- name: swiftstack/picoswiftstack
alias: swiftstack-picoswiftstack
.pg_template: &pg_definition
<<: *tests_template
needs:
......@@ -67,6 +69,8 @@ variables:
alias: postgres
- name: rroemhild/test-openldap
alias: rroemhild-test-openldap
- name: swiftstack/picoswiftstack
alias: swiftstack-picoswiftstack
.mysql_template: &mysql_definition
<<: *tests_template
needs:
......@@ -76,6 +80,8 @@ variables:
alias: mariadb
- name: rroemhild/test-openldap
alias: rroemhild-test-openldap
- name: swiftstack/picoswiftstack
alias: swiftstack-picoswiftstack
### Publish tag changelog and create a toot
##
......
......@@ -30,6 +30,7 @@
- TECH'advantage, https://www.tech-advantage.com/ (paid for LDAP invitations feature)
- pi2 (typo)
- Rain (bugfix)
- Nicolas Constant (notifications)
## Vulnerabilities / bug hunters πŸ›
......
Revision history for Lufi
?.??.? ????-??-??
- Notifications when uploading and downloading files (#181)
- Use Weblate instead of Zanata for translations (https://weblate.framasoft.org/projects/lufi/development/)
- Add config API endpoint (#183)
- Show latest tag and commit of the instance in about page and config API endpoint (#174)
- Add support for Swift object storage (EXPERIMENTAL)
0.04.6 2019-11-07
- Now can send large files (>2Gio) while using a DB other than SQLite (#165)
......
EXTRACTDIR=-D lib -D themes/default/templates
POT=themes/default/lib/Lufi/I18N/lufi.pot
ENPO=themes/default/lib/Lufi/I18N/en.po
XGETTEXT=carton exec local/bin/xgettext.pl -u
CARTON=carton exec
REAL_LUFI=script/application
......@@ -7,18 +8,10 @@ LUFI=script/lufi
locales:
$(XGETTEXT) $(EXTRACTDIR) -o $(POT) 2>/dev/null
push-locales: locales
zanata-cli -q -B push --errors --project-version `git branch | grep \* | cut -d ' ' -f2-`
pull-locales:
zanata-cli -q -B pull --min-doc-percent 50 --project-version `git branch | grep \* | cut -d ' ' -f2-`
stats-locales:
zanata-cli -q stats --project-version `git branch | grep \* | cut -d ' ' -f2-`
$(XGETTEXT) $(EXTRACTDIR) -o $(ENPO) 2>/dev/null
podcheck:
podchecker lib/Lufi/DB/File.pm lib/Lufi/DB/Slice.pm
podchecker lib/Lufi/DB/File.pm lib/Lufi/DB/Slice.pm lib/Lufi/DB/Invitation.pm
cover:
PERL5OPT='-Ilib/' HARNESS_PERL_SWITCHES='-MDevel::Cover' $(CARTON) cover --ignore_re '^local'
......@@ -37,6 +30,14 @@ ldap:
ldapdev: ldap dev
swift:
sudo docker run -d --rm -p 8080:8080 --hostname="picoswiftstack" --name="picoswiftstack" swiftstack/picoswiftstack; exit 0
@echo "Sleeping 20 seconds to let picoswiftstack start"
@sleep 20
sudo docker exec picoswiftstack get_auth
swiftdev: swift dev
devlog:
multitail log/development.log
......
......@@ -6,9 +6,9 @@ requires 'Mojolicious::Plugin::Mail';
requires 'Mojolicious::Plugin::GzipStatic';
requires 'Mojolicious::Plugin::StaticCache';
requires 'Mojolicious::Plugin::CSPHeader', '>= 0.06';
requires 'Mojolicious::Plugin::FiatTux::Helpers', '== 0.10', url => 'https://framagit.org/fiat-tux/mojolicious/mojolicious-plugin-fiattux-helpers/-/archive/0.10/mojolicious-plugin-fiattux-helpers-0.10.tar.gz';
requires 'Mojolicious::Plugin::FiatTux::GrantAccess', '== 0.06', url => 'https://framagit.org/fiat-tux/mojolicious/mojolicious-plugin-fiattux-grantaccess/-/archive/0.06/mojolicious-plugin-fiattux-grantaccess-0.06.tar.gz';
requires 'Mojolicious::Plugin::FiatTux::Themes', '== 0.02', url => 'https://framagit.org/fiat-tux/mojolicious/mojolicious-plugin-fiattux-themes/-/archive/0.02/mojolicious-plugin-fiattux-themes-0.02.tar.gz';
requires 'Mojolicious::Plugin::FiatTux::Helpers', '== 0.12', url => 'https://framagit.org/fiat-tux/mojolicious/fiat-tux/mojolicious-plugin-fiattux-helpers/-/archive/0.12/mojolicious-plugin-fiattux-helpers-0.12.tar.gz';
requires 'Mojolicious::Plugin::FiatTux::GrantAccess', '== 0.07', url => 'https://framagit.org/fiat-tux/mojolicious/fiat-tux/mojolicious-plugin-fiattux-grantaccess/-/archive/0.07/mojolicious-plugin-fiattux-grantaccess-0.07.tar.gz';
requires 'Mojolicious::Plugin::FiatTux::Themes', '== 0.02', url => 'https://framagit.org/fiat-tux/mojolicious/fiat-tux/mojolicious-plugin-fiattux-themes/-/archive/0.02/mojolicious-plugin-fiattux-themes-0.02.tar.gz';
requires 'Filesys::DiskUsage';
requires 'Switch';
requires 'Locale::Maketext';
......@@ -55,3 +55,6 @@ feature 'mysql', 'MySQL support' => sub {
requires 'Mojo::mysql';
requires 'Mojolicious::Plugin::PgURLHelper';
};
feature 'swift-storage', 'Openstack Swift object storage support' => sub {
requires 'Net::OpenStack::Swift';
};
This diff is collapsed.
......@@ -66,6 +66,12 @@ sub startup {
# Helpers
$self->plugin('Lufi::Plugin::Helpers');
# Now helpers has been loaded, time to check Swift container
if ($config->{swift}) {
$self->check_swift_container();
$self->log->info('EXPERIMENTAL Using Swift object storage');
}
# Recurrent task
Mojo::IOLoop->recurring(2 => sub {
my $loop = shift;
......@@ -74,8 +80,10 @@ sub startup {
});
# Create directory if needed
mkdir($self->config('upload_dir'), 0700) unless (-d $self->config('upload_dir'));
die ('The upload directory ('.$self->config('upload_dir').') is not writable') unless (-w $self->config('upload_dir'));
if (!defined($config->{swift})) {
mkdir($self->config('upload_dir'), 0700) unless (-d $self->config('upload_dir'));
die ('The upload directory ('.$self->config('upload_dir').') is not writable') unless (-w $self->config('upload_dir'));
}
# Configure sessions
my $sessions = Mojolicious::Sessions->new;
......@@ -159,6 +167,11 @@ sub startup {
->to('Misc#about')
->name('about');
# About config API endpoint
$r->get('/about/config')
->to('Misc#config_infos')
->name('config');
# Generated js files
$r->get('/partial/:file')
->to('Misc#js_files')
......
package Lufi::Command::copyFilesToSwift;
use Mojo::Base 'Mojolicious::Command';
use File::Spec;
use Term::ProgressBar;
has description => 'Copy files from filesystem to Swift object storage';
has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
if ($c->app->config('swift')) {
$c->app->check_swift_container();
my @dirs = glob(File::Spec->catdir($c->app->config('upload_dir'), '*'));
unless (scalar(@dirs)) {
say sprintf('The configured upload_dir (%s) seems to be empty. Is `upload_dir` configured in lufi.conf?', $c->app->config('upload_dir'));
exit 1;
}
say sprintf('%d folders to upload to Swift (can\'t say how many files, or the total size, sorry). This can take some time.', scalar(@dirs));
print 'Do you want to continue? [Y/n] ';
my $confirm = <STDIN>;
if ($confirm =~ m/yes|y/i) {
my $progress = Term::ProgressBar->new({ count => scalar(@dirs), ETA => 'linear', name => 'Copying to Swift'});
for my $dir (@dirs) {
my @files = glob(File::Spec->catfile($dir, '*'));
for my $file (@files) {
my ($volume, $directories, $filename) = File::Spec->splitpath($file);
my @file_dirs = File::Spec->splitdir($directories);
my $short = ($file_dirs[-1] ne '') ? $file_dirs[-1] : $file_dirs[-2];
my $path = File::Spec->catfile($short, $filename);
open my $fh, '<', $file or die sprintf('Unable to open file %s: %s', $file, $!);
$c->app->swift->put_object(
container_name => $c->app->config('swift')->{container},
object_name => $path,
content_length => -s $file,
content => $fh
);
close $fh;
}
$progress->update();
}
say sprintf('The copy to Swift object storage has ended. You can test Lufi, then delete `%s` directory', $c->app->config('upload_dir'));
} else {
say 'You want to stop. No problem.';
}
} else {
say 'You didn\'t configure `swift` in your config file. Exiting.';
exit 1;
}
}
=encoding utf8
=head1 NAME
Lufi::Command::copyFilesToSwift Copy files from filesystem to Swift object storage
=head1 SYNOPSIS
Usage: script/lufi copyFilesToSwift
This command needs you to:
- set `upload_dir` in your config file (otherwise, it will use the default path, `files` to copy files from)
- configure `swift` with correct informations in your config file
=cut
1;
package Lufi::Command::sqliteToOtherDB;
use Mojo::Base 'Mojolicious::Command';
use Lufi::DB::BreakingChange;
use Lufi::DB::File;
use Lufi::DB::Slice;
use Lufi::DB::Invitation;
......@@ -9,7 +10,7 @@ use Term::ProgressBar;
use Lufi::DefaultConfig qw($default_config);
has description => 'Migrate the records from a SQLite db to the currently configured database';
has usage => sub { shift->extract_usage };
has usage => sub { shift->extract_usage };
sub run {
my $c = shift;
......@@ -36,8 +37,9 @@ sub run {
my $files = $sqlite->db->select('files', undef)->hashes;
my $slices = $sqlite->db->select('slices', undef)->hashes;
my $invitations = $sqlite->db->select('invitations', undef)->hashes;
my $changes = $sqlite->db->select('breakingchanges', undef)->hashes;
my $progress = Term::ProgressBar->new({count => $files->size + $slices->size + $invitations->size});
my $progress = Term::ProgressBar->new({count => $files->size + $slices->size + $invitations->size + $changes->size});
$files->each(sub {
my ($file, $num) = @_;
......@@ -69,7 +71,6 @@ sub run {
Lufi::DB::Slice->new(app => $c->app)
->short($slice->{short})
->j($slice->{j})
->path($slice->{path})
->write();
$progress->update();
......@@ -92,13 +93,22 @@ sub run {
->write();
$progress->update();
});
$changes->each(sub {
my ($change, $num) = @_;
Lufi::DB::BreakingChange->new(app => $c->app)
->change($change->{change})
->ack($change->{ack})
->write();
$progress->update();
});
}
=encoding utf8
=head1 NAME
Lufi::Command::cron::sqliteToOtherDB Migrate the records from a SQLite db to the currently configured database
Lufi::Command::sqliteToOtherDB Migrate the records from a SQLite db to the currently configured database
=head1 SYNOPSIS
......
......@@ -2,7 +2,7 @@
package Lufi::Command::theme;
use Mojo::Base 'Mojolicious::Commands';
use FindBin qw($Bin);
use File::Spec qw(catfile cat dir);
use File::Spec qw(catfile catdir);
use File::Path qw(make_path);
has description => 'Create new theme skeleton.';
......
......@@ -100,7 +100,8 @@ sub upload {
}
}
# Check that we have enough space (multiplying by 2 since it's encrypted, it takes more place that the original file)
if ($json->{part} == 0 && ($json->{size} * 2) >= dfportable($c->config('upload_dir'))->{bavail}) {
# Only check if using filesystem, not Swift storage
if (!defined($c->config('swift')) && $json->{part} == 0 && ($json->{size} * 2) >= dfportable($c->config('upload_dir'))->{bavail}) {
$stop = 1;
return $ws->send(decode('UTF-8', encode_json(
{
......@@ -190,19 +191,12 @@ sub upload {
# If we already have a part, it's a resend because the websocket has been broken
# In this case, we don't need to rewrite the file
unless ($f->slices->grep(sub { $_->j == $json->{part} })->size) {
# Create directory
my $dir = catdir($c->config('upload_dir'), $f->short);
mkdir($dir, 0700) unless (-d $dir);
# Create slice file
my $file = catfile($dir, $json->{part}.'.part');
my $s = Lufi::DB::Slice->new(
app => $c->app,
short => $f->short,
j => $json->{part},
path => $file
);
Mojo::File->new($file)->spurt($text);
j => $json->{part}
)->store($text);
push @{$f->slices}, $s;
$s->write;
......@@ -327,7 +321,7 @@ sub download {
# Get the slice
my $e = $f->slices->[$num];
my $text = Mojo::File->new($e->path)->slurp;
my $text = $e->retrieve();
my ($json2) = split('XXMOJOXX', $text, 2);
$json2 = decode 'UTF-8', $json2;
......
......@@ -2,6 +2,7 @@
package Lufi::Controller::Misc;
use Mojo::Base 'Mojolicious::Controller';
use Mojo::File;
use Mojo::JSON qw(true false);
use Mojo::URL;
use Lufi::DB::File;
......@@ -31,7 +32,34 @@ sub change_lang {
}
sub about {
shift->render(template => 'about');
my $c = shift;
$c->render(
template => 'about',
version => $c->git_version
);
}
sub config_infos {
my $c = shift;
$c->render(
json => {
report => $c->config('report'),
instance_name => $c->config('instance_name'),
max_file_size => $c->config('max_file_size'),
broadcast_message => $c->config('broadcast_message'),
default_delay => $c->config('default_delay'),
max_delay => $c->config('max_delay'),
delay_for_size => $c->config('delay_for_size'),
allow_pwd_on_files => $c->config('allow_pwd_on_files'),
force_burn_after_reading => $c->config('force_burn_after_reading'),
keep_ip_during => $c->config('keep_ip_during'),
stop_upload => (-f 'stop-upload' || -f 'stop-upload.manual') ? true : false,
need_authentication => (defined($c->config('ldap')) || defined($c->config('htpasswd'))) ? true : false,
version => $c->git_version
}
);
}
sub js_files {
......
......@@ -148,13 +148,37 @@ sub delete {
$c->slices->each(sub {
my ($e, $num) = @_;
unlink $e->path;
$e->delete_file();
});
rmdir Mojo::File->new($c->app->config('upload_dir'), $c->short);
$c->deleted(1);
$c->delete_path
->deleted(1)
->write;
$c->write;
return $c;
}
=head2 delete_path
=over 1
=item B<Usage> : C<$c-E<gt>delete_path()>
=item B<Arguments> : none
=item B<Purpose> : delete the directory of the slices on filesystem or Swift object storage
=item B<Returns> : the db accessor object
=back
=cut
sub delete_path {
my $c = shift;
if (!defined($c->app->config('swift'))) {
rmdir Mojo::File->new($c->app->config('upload_dir'), $c->short);
}
return $c;
}
......
# vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
package Lufi::DB::Slice;
use Mojo::Base -base;
use Encode 'encode';
use File::Spec::Functions;
use Mojo::Collection 'c';
has 'short';
has 'j';
has 'path';
has 'record' => 0;
has 'app';
=encoding utf8
=head1 NAME
Lufi::DB::Slice - DB abstraction layer for Lufi file
......@@ -28,8 +31,6 @@ Have a look at Lufi::DB::Slice::SQLite's code: it's simple and may be more under
=item B<j> : integer
=item B<path> : string
=item B<app> : A mojolicious object
=back
......@@ -96,15 +97,121 @@ sub write {
my $c = shift;
if ($c->record) {
$c->app->dbi->db->query('UPDATE slices SET short = ?, j = ?, path = ? WHERE short = ? AND j = ?', $c->short, $c->j, $c->path, $c->short, $c->j);
$c->app->dbi->db->query('UPDATE slices SET short = ?, j = ? WHERE short = ? AND j = ?', $c->short, $c->j, $c->short, $c->j);
} else {
$c->app->dbi->db->query('INSERT INTO slices (short, j, path) VALUES (?, ?, ?)', $c->short, $c->j, $c->path);
$c->app->dbi->db->query('INSERT INTO slices (short, j) VALUES (?, ?)', $c->short, $c->j);
$c->record(1);
}
return $c;
}
=head2 store
=over 1
=item B<Usage> : C<$c-E<gt>store($text)>
=item B<Arguments> : a scalar value
=item B<Purpose> : will store the content to the object's path, either on filesystem or on Swift object storage
=item B<Returns> : the db accessor object
=back
=cut
sub store {
my $c = shift;
my $text = shift;
if ($c->app->config('swift')) {
$c->app->swift->put_object(
container_name => $c->app->config('swift')->{container},
object_name => $c->get_path(),
content_length => length(Encode::encode_utf8($text)),
content => Encode::encode_utf8($text)
);
} else {
# Create directory
my $dir = catfile($c->app->config('upload_dir'), $c->short);
mkdir($dir, 0700) unless (-d $dir);
# Write file
my $file = catfile($c->app->config('upload_dir'), $c->get_path());
Mojo::File->new($file)->spurt($text);
}
return $c;
}
=head2 retrieve
=over 1
=item B<Usage> : C<$c-E<gt>retrieve>
=item B<Arguments> : none
=item B<Purpose> : get file from storage, either filesystem or Swift object storage
=item B<Returns> : the data from the file
=back
=cut
sub retrieve {
my $c = shift;
my $upload = shift;
if ($c->app->config('swift')) {
my $file;
$c->app->swift->get_object(
container_name => $c->app->config('swift')->{container},
object_name => $c->get_path(),
write_code => sub {
my ($status, $message, $headers, $chunk) = @_;
$file .= $chunk;
}
);
return Encode::decode_utf8($file);
} else {
my $file = catfile($c->app->config('upload_dir'), $c->get_path());
return Mojo::File->new($file)->slurp;
}
}
=head2 delete_file
=over 1
=item B<Usage> : C<$c-E<gt>delete_file()>
=item B<Arguments> : none
=item B<Purpose> : delete the file on filesystem or Swift object storage
=item B<Returns> : the db accessor object
=back
=cut
sub delete_file {
my $c = shift;
if ($c->app->config('swift')) {
$c->app->swift->delete_object({
container_name => $c->app->config('swift')->{container},
object_name => $c->get_path()
});
} else {
my $file = catfile($c->app->config('upload_dir'), $c->get_path());
unlink $file or warn sprintf('Could not unlink %s: %s', $file, $!);
}
return $c;
}
=head2 get_slices_of_file
=over 1
......@@ -147,7 +254,7 @@ sub get_slices_of_file {
=item B<Arguments> : none
=item B<Purpose> : delete all file records from database unconditionnally
=item B<Purpose> : delete all slices records from database unconditionnally
=item B<Returns> : nothing
......@@ -161,6 +268,50 @@ sub delete_all {
$c->app->dbi->db->delete('slices');
}
=head2 path
=over 1
=item B<Usage> : C<$c-E<gt>path()>
=item B<Arguments> : non
=item B<Purpose> : format the path of the file, relative to the directory of the Swift object storage
=item B<Returns> : the path of the file
=back
=cut
sub get_path {
my $c = shift;
return catfile($c->short, sprintf('%d.part', $c->j));
}
=head2 count
=over 1
=item B<Usage> : C<$c-E<gt>count()>
=item B<Arguments> : none
=item B<Purpose> : get count of slices records from database
=item B<Returns> : integer
=back
=cut
sub count {
my $c = shift;
return $c->app->dbi->db->query('SELECT count(*) AS count FROM slices')->hashes->first->{count};
}
=head2 _slurp
=over 1
......@@ -195,7 +346,6 @@ sub _slurp {
if ($slice) {
$c->short($slice->{short});
$c->j($slice->{j});
$c->path($slice->{path});
$c->record(1);
}
......