From c44cbc889125f933ffa2c2f0b830da03d3752121 Mon Sep 17 00:00:00 2001 From: b Date: Tue, 18 Mar 2025 02:04:54 +0100 Subject: [PATCH] improve uploader, diff --- upload.1.pl | 172 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 128 insertions(+), 44 deletions(-) diff --git a/upload.1.pl b/upload.1.pl index 400678a..8e187f6 100644 --- a/upload.1.pl +++ b/upload.1.pl @@ -29,7 +29,8 @@ use botm_common ( 'join_path', 'read_data_file', 'write_data_file', 'system_encoded', '_x_encoded', - 'merge_settings' + 'merge_settings', + 'url_encode', ); binmode STDIN, ':encoding(UTF-8)'; @@ -57,7 +58,8 @@ my %state; my $target; my $server; my $current_frame; -my $last_frame; +my $last_frame; # not needed? +my $diff = 0; my @upload_list; print $time." UPLOAD\n"; @@ -85,29 +87,29 @@ $server = $remote{'user'}.'@'.$remote{'server'}; $target = $remote{$target }; %settings = read_data_file(DATA_SETTINGS_PATH()); -if (%settings == 0) { +if (scalar(%settings) == 0) { print STDERR "didn't read settings\n"; print "FAIL\n"; exit 1; } %default = read_data_file(DATA_DEFAULT_PATH()); -if (%default == 0) { +if (scalar(%default) == 0) { print STDERR "didn't read default\n"; print "FAIL\n"; exit 1; } %state = read_data_file( - ['ssh', $server, '-t', 'cat '.join_path('/', $target, 'state').' 2>/dev/null'], '', + ['ssh', '-q', '-T', $server, '-t', 'stty raw; cat '.join_path('/', $target, 'state').' 2>/dev/null'], '', '', '', '', '-|', '' ); -if (%state == 0) { +if (scalar(%state) == 0) { print STDERR "didn't read state\n"; print "FAIL\n"; exit 1; } $current_frame = (int($state{'state'}) > 0) ? int($state{'last'}) : -1; -$last_frame = int($settings{'last'}); +$last_frame = int($settings{'last'}); # not needed? foreach my $id (@ARGV) { if ($id eq 'all') { @@ -115,8 +117,7 @@ foreach my $id (@ARGV) { upload_interface(\@upload_list); upload_default(\@upload_list); upload_noaccess(\@upload_list, 1, 1); - for (my $f=0; $f<=$last_frame; $f+=1) { - upload_frame(\@upload_list, $f, 1, 1); + for (my $f=0; upload_frame(\@upload_list, $f, 1, 1); $f+=1) { } for (my $a=0; upload_attachment(\@upload_list, $a, 1, 1); $a+=1) { } @@ -125,25 +126,28 @@ foreach my $id (@ARGV) { my $f = int($2); my $data = (($1 eq 'd') or ($1 eq '')); my $file = (($1 eq 'f') or ($1 eq '')); - if ($f > $last_frame) { - print "$f > $last_frame\n"; - $fail += 1; + my $r = upload_frame(\@upload_list, $f, $data, $file); + unless ($r) { + print '!'.$f."\n"; + $fail = 1; next; } - upload_frame(\@upload_list, $f, $data, $file); } elsif ($id =~ /^([df]?)([\-+])$/) { my $f_start = ($2 eq '+') ? ($current_frame+1) : 0; - my $f_end = ($2 eq '+') ? $last_frame : $current_frame; + my $f_end = ($2 eq '+') ? ~0 : $current_frame; my $data = (($1 eq 'd') or ($1 eq '')); my $file = (($1 eq 'f') or ($1 eq '')); for (my $f=$f_start; $f<=$f_end; $f+=1) { - if ($f > $last_frame) { - print "$f > $last_frame\n"; + my $r = upload_frame(\@upload_list, $f, $data, $file); + unless ($r) { + if (($f_end == ~0) && ($f != $f_start)) { + last; + } + print '!'.$f."\n"; $fail += 1; next; } - upload_frame(\@upload_list, $f, $data, $file); } } elsif ($id =~ /^(([df]?)([0-9]+))?([\-+])(([df]?)([0-9]+))?$/) { @@ -170,7 +174,7 @@ foreach my $id (@ARGV) { } elsif ($1 ne '') { $f_start = int($3); - $f_end = (($4 eq '-') ? $current_frame : $last_frame); + $f_end = (($4 eq '-') ? $current_frame : ~0); $df = $2; } elsif ($5 ne '') { @@ -183,11 +187,6 @@ foreach my $id (@ARGV) { $fail += 1; next; } - if ($f_start > $last_frame) { - print "$f_start > $last_frame\n"; - $fail += 1; - next; - } if ($f_start > $f_end) { print "$f_start > $f_end\n"; $fail += 1; @@ -196,19 +195,21 @@ foreach my $id (@ARGV) { $data = (($df eq 'd') or ($df eq '')); $file = (($df eq 'f') or ($df eq '')); for (my $f=$f_start; $f<=$f_end; $f+=1) { - if ($f > $last_frame) { - print "$f > $last_frame\n"; + my $r = upload_frame(\@upload_list, $f, $data, $file); + unless ($r) { + if (($f_end == ~0) && ($f != $f_start)) { + last; + } + print '!'.$f."\n"; $fail += 1; next; } - upload_frame(\@upload_list, $f, $data, $file); } } elsif ($id =~ /^[df_]$/) { my $data = (($& eq 'd') or ($& eq '_')); my $file = (($& eq 'f') or ($& eq '_')); - for (my $f=0; $f<=$last_frame; $f+=1) { - upload_frame(\@upload_list, $f, $data, $file); + for (my $f=0; upload_frame(\@upload_list, $f, $data, $file); $f+=1) { } } elsif ($id =~ /^a([df]?)([0-9]+)$/) { @@ -308,44 +309,121 @@ foreach my $id (@ARGV) { elsif ($id eq 's') { upload_settings(\@upload_list); } + elsif ($id eq 'diff') { + $diff = 1; + } else { print $id." ???\n"; $fail += 1; } - +} + +if ($diff) { + print "DIFF\n"; } foreach my $upload (@upload_list) { my $r; my $remote_server_path = $server.':'.$upload->{'remote_path'}; - print "\n".$upload->{'local_path'}.' -> '.$remote_server_path."\n"; + + if ($diff) { + unless ($upload->{'special'}) { + next; + } + my %local_data = read_data_file($upload->{'local_path'}); + my %remote_data = read_data_file( + ['ssh', '-q', '-T', $server, '-t', 'stty raw; cat '.$upload->{'remote_path'}.' 2>/dev/null'], '', + '', '', '', + '-|', '' + ); + if ((scalar(%local_data) == 0) and (scalar(%remote_data) != 0)) { + print "\n! ".$upload->{'local_path'}."\n"; + next; + } + elsif ((scalar(%local_data) != 0) and (scalar(%remote_data) == 0)) { + print "\n! ".$remote_server_path."\n"; + next; + } + elsif ((scalar(%local_data) == 0) and (scalar(%remote_data) == 0)) { + next; + } + if (ref($upload->{'preserve'})) { + foreach my $key (@{$upload->{'preserve'}}) { + if (exists $local_data{$key}) { + delete $local_data{$key}; + } + if (exists $remote_data{$key}) { + delete $remote_data{$key}; + } + } + } + my %keys; + my @differences; + foreach my $key (keys %local_data) { + $keys{$key} = 1; + } + foreach my $key (keys %remote_data) { + $keys{$key} = 1; + } + foreach my $key (keys %keys) { + unless (exists $local_data{$key}) { + push(@differences, '--> '.$key); + next; + } + unless (exists $remote_data{$key}) { + push(@differences, '<-- '.$key); + next; + } + unless ($local_data{$key} eq $remote_data{$key}) { + push(@differences, '=/= '.$key); + push(@differences, '<== '.$local_data{$key}.' <=='); + push(@differences, '==> '.$remote_data{$key}.' ==>'); + push(@differences, '=\='); + next; + } + } + if (scalar(@differences) != 0) { + print "\n".$upload->{'local_path'}.' =/= '.$remote_server_path."\n"; + foreach my $line (@differences) { + print $line."\n"; + } + } + next; + } + print "\n".$upload->{'local_path'}.' --> '.$remote_server_path."\n"; if ($upload->{'special'}) { - my %data = read_data_file($upload->{'local_path'}); - if (%data == 0) { + if ($upload->{'frame'} eq '') { + $upload->{'frame'} = ~0; + } + my %local_data = read_data_file($upload->{'local_path'}); + if (scalar(%local_data) == 0) { print STDERR 'didn\'t read '.$upload->{'local_path'}."\n"; print "FAIL\n"; $fail += 1; next; } my %remote_data = read_data_file( - ['ssh', $server, '-t', 'cat '.$upload->{'remote_path'}.' 2>/dev/null'], '', + ['ssh', '-q', '-T', $server, '-t', 'stty raw; cat '.$upload->{'remote_path'}.' 2>/dev/null'], '', '', '', '', '-|', '' ); - if ((%remote_data == 0) && ($upload->{'frame'} <= $current_frame)) { + if ((scalar(%remote_data) == 0) && ($upload->{'frame'} <= $current_frame)) { print STDERR 'didn\'t read '.$remote_server_path."\n"; print "FAIL\n"; $fail += 1; next; } - foreach my $key ('ongtime', 'timer', 'ott') { - if (defined $remote_data{$key}) { - $data{$key} = $remote_data{$key}; + my %new_data = %local_data; + if (ref($upload->{'preserve'})) { + foreach my $key (@{$upload->{'preserve'}}) { + if (defined $remote_data{$key}) { + $new_data{$key} = $remote_data{$key}; + } } } $r = write_data_file( - ['ssh', $server, '-t', 'cat > '.$upload->{'remote_path'}], \%data, '', + ['ssh', '-q', '-T', $server, '-t', 'stty raw; cat > '.$upload->{'remote_path'}], \%new_data, '', '', '', '', '|-', '' ); @@ -376,7 +454,7 @@ sub upload_settings { push @$upload_list, { 'local_path' => DATA_SETTINGS_PATH(), 'remote_path' => join_path('/', $target, 'settings'), - 'special' => 0 + 'special' => 1 }; } @@ -385,7 +463,7 @@ sub upload_default { push @$upload_list, { 'local_path' => DATA_DEFAULT_PATH(), 'remote_path' => join_path('/', $target, 'default'), - 'special' => 0 + 'special' => 1 }; } @@ -395,7 +473,7 @@ sub upload_noaccess { push @$upload_list, { 'local_path' => DATA_NOACCESS_PATH(), 'remote_path' => join_path('/', $target, 'noaccess'), - 'special' => 0 + 'special' => 1 }; } if ($file) { @@ -414,12 +492,16 @@ sub upload_frame { (my $upload_list, my $id, my $data, my $file) = @_; $id = int($id); my $path = join_path('/', DATA_PATH(), $id); + unless (_x_encoded('-f', $path)) { + return 0; + } if ($data) { push @$upload_list, { 'local_path' => $path, 'remote_path' => join_path('/', $target, $id), 'special' => 1, - 'frame' => $id + 'frame' => $id, + 'preserve' => ['ongtime', 'timer', 'ott'] }; } if ($file) { @@ -436,6 +518,7 @@ sub upload_frame { 'special' => 0 }; } + return 1; } sub upload_attachment { @@ -445,15 +528,16 @@ sub upload_attachment { unless (_x_encoded('-f', $path)) { return 0; } + my %attachment_data = read_data_file($path); if ($data) { push @$upload_list, { 'local_path' => $path, 'remote_path' => join_path('/', $target, $id), - 'special' => 0 + 'special' => 1, + 'frame' => int($attachment_data{'frame'}), }; } if ($file) { - my %attachment_data = read_data_file($path); unless (($attachment_data{'filename'} eq '') or ($attachment_data{'content'} ne '')) { $file = $attachment_data{'filename'}; push @$upload_list, { -- 2.30.2