zydronium/imapsync
1
0
Fork 0
mirror of https://github.com/imapsync/imapsync.git synced 2025-06-05 20:27:20 +02:00
Code Issues Projects Releases Packages Wiki Activity

1.285

Browse source
This commit is contained in:
Nick Bebout 2011-03-12 02:44:48 +00:00
parent 9ca0e338a4
commit 5cdcc66e9c
11 changed files with 297 additions and 2862 deletions
Download patch file Download diff file Expand all files Collapse all files

2
BUG_IMAPClient_3.xx
View file

@ -18,4 +18,6 @@ Fixed in Mail-IMAPClient-3.10/
Wrong. Lacks isUnconnected() method.
5) Mail-IMAPClient-3.19 is a good one.
No known bug

23
ChangeLog
View file

@ -1,17 +1,32 @@
RCS file: RCS/imapsync,v
Working file: imapsync
head: 1.284
head: 1.285
branch:
locks: strict
gilles: 1.284
gilles: 1.285
access list:
symbolic names:
keyword substitution: kv
total revisions: 284; selected revisions: 284
total revisions: 285; selected revisions: 285
description:
----------------------------
revision 1.284 locked by: gilles;
revision 1.285 locked by: gilles;
date: 2009/07/03 00:59:20; author: gilles; state: Exp; lines: +38 -17
Applied "patch -p0 < patches/imapsync.1.284.patch"
Attached is a patch against 1.284 with the following changes:
- [Bug] to_folder_name() failed to escape $f_prefix so --prefix1 <string> was treated like --prefix1 <regex> instead
- [Enhancement] updated parse_header_msg1() to return 0 on duplicate or undef on "no header" so now the "+ Skipping msg" will show either "no header" or "duplicate" as appropriate
- [Bug] flag handling: make sure we don't try to set \Recent as RFC 2060 doesn't allow for that
- [Enhancement] flag handling: only add missing flags instead of always calling $to->store() to set flags even when the flags already matched what we wanted... my first patch just used string comparison, this patch actually uses a hash for comparison so we're smarter about flags even when the order on $from and $to are different now - definitely helps speed up imapsync when lots of messages have flags and running sync in multiple passes
Phil
----------------------------
revision 1.284
date: 2009/06/30 03:14:24; author: gilles; state: Exp; lines: +15 -19
allow Mail::IMAPClient 3.0.xx by default
Removed Mail::IMAPClient::Ssl since 3.0.19 has it now.

6
Makefile
View file

@ -1,5 +1,5 @@
# $Id: Makefile,v 1.22 2009/06/30 04:33:25 gilles Exp gilles $
# $Id: Makefile,v 1.23 2009/07/03 01:01:13 gilles Exp gilles $
TARGET=imapsync
@ -69,7 +69,7 @@ VERSION: $(TARGET) Makefile
clean: clean_tilde clean_man
clean_test:
rm -f .test .test_3xx
rm -f .test .test_3xx .test_229
clean_tilde:
rm -f *~
@ -127,7 +127,7 @@ clean_dist:
.PHONY: lfo niouze
lfo: dist niouze_lfo lfo_upload niouze
lfo: dist lfo_upload niouze_lfo niouze
lfo_upload:
rsync -avH --delete . \

4
README
View file

@ -3,7 +3,7 @@ NAME
Synchronise mailboxes between two imap servers. Good at IMAP migration.
More than 32 different IMAP server softwares supported with success.
$Revision: 1.284 $
$Revision: 1.285 $
INSTALL
imapsync works fine under any Unix OS with perl.
@ -362,5 +362,5 @@ SIMILAR SOFTWARES
Feedback (good or bad) will always be welcome.
$Id: imapsync,v 1.284 2009/06/30 03:14:24 gilles Exp gilles $
$Id: imapsync,v 1.285 2009/07/03 00:59:20 gilles Exp gilles $

2
VERSION
View file

@ -1 +1 @@
1.284
1.285

4
i Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
perl -IMail-IMAPClient-2.2.9 ./imapsync "$@"

53
imapsync
View file

@ -9,7 +9,7 @@ tool. Synchronise mailboxes between two imap servers. Good
at IMAP migration. More than 32 different IMAP server softwares
supported with success.
$Revision: 1.284 $
$Revision: 1.285 $
=head1 INSTALL
@ -418,7 +418,7 @@ Entries for imapsync:
Feedback (good or bad) will always be welcome.
$Id: imapsync,v 1.284 2009/06/30 03:14:24 gilles Exp gilles $
$Id: imapsync,v 1.285 2009/07/03 00:59:20 gilles Exp gilles $
=cut
@ -484,7 +484,7 @@ my(
use vars qw ($opt_G); # missing code for this will be option.
$rcs = '$Id: imapsync,v 1.284 2009/06/30 03:14:24 gilles Exp gilles $ ';
$rcs = '$Id: imapsync,v 1.285 2009/07/03 00:59:20 gilles Exp gilles $ ';
$rcs =~ m/,v (\d+\.\d+)/;
$VERSION = ($1) ? $1: "UNKNOWN";
@ -547,8 +547,8 @@ while (@argv_copy) {
my $banner = join("",
'$RCSfile: imapsync,v $ ',
'$Revision: 1.284 $ ',
'$Date: 2009/06/30 03:14:24 $ ',
'$Revision: 1.285 $ ',
'$Date: 2009/07/03 00:59:20 $ ',
"\n",localhost_info(),
" and the module Mail::IMAPClient version used here is ",
$VERSION_IMAPClient,"\n",
@ -1237,7 +1237,7 @@ sub to_folder_name {
my ($t_fold);
my ($x_fold) = @_;
# first we remove the prefix
$x_fold =~ s/^$f_prefix//;
$x_fold =~ s/^\Q$f_prefix\E//;
$debug and print "removed source prefix: [$x_fold]\n";
$t_fold = separator_invert($x_fold,$f_sep, $t_sep);
$debug and print "inverted separators: [$t_fold]\n";
@ -1406,9 +1406,11 @@ FOLDER: foreach my $f_fold (@f_folders) {
last FOLDER if $from->IsUnconnected();
foreach my $m (@f_msgs) {
unless (parse_header_msg1($from, $m, $f_heads, $f_fir, "F", \%f_hash)) {
my $rc = parse_header_msg1($from, $m, $f_heads, $f_fir, "F", \%f_hash);
if (!$rc) {
my $reason = !defined($rc) ? "no header" : "duplicate";
my $f_size = $f_fir->{$m}->{"RFC822.SIZE"} || 0;
print "+ Skipping msg #$m:$f_size in folder $f_fold (no header so we ignore this message)\n";
print "+ Skipping msg #$m:$f_size in folder $f_fold ($reason so we ignore this message)\n";
$mess_size_total_skipped += $f_size;
$mess_skipped += 1;
}
@ -1427,7 +1429,14 @@ FOLDER: foreach my $f_fold (@f_folders) {
$debug and print "Time fir: ", timenext(), " s\n";
last FOLDER if $to->IsUnconnected();
foreach my $m (@t_msgs) {
parse_header_msg1($to, $m, $t_heads, $t_fir, "T", \%t_hash);
my $rc = parse_header_msg1($to, $m, $t_heads, $t_fir, "T", \%t_hash);
if (!$rc) {
my $reason = !defined($rc) ? "no header" : "duplicate";
my $t_size = $t_fir->{$m}->{"RFC822.SIZE"} || 0;
print "+ Skipping msg #$m:$t_size in 'to' folder $t_fold ($reason so we ignore this message)\n";
#$mess_size_total_skipped += $msize;
#$mess_skipped += 1;
}
}
$debug and print "Time headers: ", timenext(), " s\n";
@ -1572,7 +1581,7 @@ FOLDER: foreach my $f_fold (@f_folders) {
my $flags_f = $f_hash{$m_id}{'F'} || "";
# RFC 2060: This flag can not be altered by any client
$flags_f =~ s@\\Recent@@gi;
$flags_f =~ s@\\Recent\s?@@gi;
$flags_f = flags_regex($flags_f) if @regexflag;
my $new_id;
@ -1638,14 +1647,24 @@ FOLDER: foreach my $f_fold (@f_folders) {
my $flags_f = $f_hash{$m_id}{'F'} || "";
my $flags_t = $t_hash{$m_id}{'F'} || "";
# No flag \Recent here, no ?
# RFC 2060: This flag can not be altered by any client
$flags_f =~ s@\\Recent\s?@@gi;
$flags_f = flags_regex($flags_f) if @regexflag;
$debug and print "Setting flags from($flags_f) to($flags_t)\n";
# compare flags - add missing flags
my @ff = split(' ', $flags_f );
my %ft = map { $_ => 1 } split(' ', $flags_t );
my @flags_a = map { exists $ft{$_} ? () : $_ } @ff;
# This add or change flags but no flag are removed with this
$to->store($t_msg, "+FLAGS.SILENT ($flags_f)" )
if (!$dry and $flags_f ne $flags_t);
$debug and print "Setting flags(@flags_a) ffrom($flags_f) fto($flags_t) on msg #$t_msg in $t_fold\n";
# This adds or changes flags but no flag are removed with this
if (!$dry and @flags_a and !$to->store($t_msg, "+FLAGS.SILENT (@flags_a)") ) {
warn "Could not add flags '@flags_a' flagf '$flags_f'",
" flagt '$flags_t' on msg #$t_msg in $t_fold: ",
$to->LastError, "\n";
#$error++;
}
last FOLDER if $to->IsUnconnected();
$debug and do {
@ -1971,7 +1990,7 @@ sub parse_header_msg1 {
#$headstr = $imap->message_string($m_uid);
print "no header so we ignore this message\n";
return;
return undef;
}
my $size = $s_fir->{$m_uid}->{"RFC822.SIZE"};
my $flags = $s_fir->{$m_uid}->{"FLAGS"};
@ -1986,6 +2005,8 @@ sub parse_header_msg1 {
else {
$key = "$m_md5:$size";
}
# 0 return code is used to identify duplicate message hash
return 0 if exists $s_hash->{"$key"};
$s_hash->{"$key"}{'5'} = $m_md5;
$s_hash->{"$key"}{'s'} = $size;
$s_hash->{"$key"}{'D'} = $idate;

2819
imapsync.282
View file

File diff suppressed because it is too large Load diff

108
patches/imapsync.1.282.patch2 Normal file
View file

@ -0,0 +1,108 @@
--- imapsync.1.282+patch1 2009-06-19 15:44:09.140625000 -0400
+++ imapsync 2009-06-19 15:50:31.437500000 -0400
@@ -515,7 +515,7 @@
if (@_) { $self->{SSL} = shift }
return $self->{SSL};
-};
+} unless Mail::IMAPClient->can("Ssl");
return(1);
}else{
@@ -1245,7 +1245,7 @@
my ($t_fold);
my ($x_fold) = @_;
# first we remove the prefix
- $x_fold =~ s/^$f_prefix//;
+ $x_fold =~ s/^\Q$f_prefix\E//;
$debug and print "removed source prefix: [$x_fold]\n";
$t_fold = separator_invert($x_fold,$f_sep, $t_sep);
$debug and print "inverted separators: [$t_fold]\n";
@@ -1414,9 +1414,11 @@
last FOLDER if $from->IsUnconnected();
foreach my $m (@f_msgs) {
- unless (parse_header_msg1($from, $m, $f_heads, $f_fir, "F", \%f_hash)) {
+ my $rc = parse_header_msg1($from, $m, $f_heads, $f_fir, "F", \%f_hash);
+ if (!$rc) {
+ my $reason = !defined($rc) ? "no header" : "duplicate";
my $f_size = $f_fir->{$m}->{"RFC822.SIZE"} || 0;
- print "+ Skipping msg #$m:$f_size in folder $f_fold (no header so we ignore this message)\n";
+ print "+ Skipping msg #$m:$f_size in folder $f_fold ($reason so we ignore this message)\n";
$mess_size_total_skipped += $f_size;
$mess_skipped += 1;
}
@@ -1435,7 +1437,14 @@
$debug and print "Time fir: ", timenext(), " s\n";
last FOLDER if $to->IsUnconnected();
foreach my $m (@t_msgs) {
- parse_header_msg1($to, $m, $t_heads, $t_fir, "T", \%t_hash);
+ my $rc = parse_header_msg1($to, $m, $t_heads, $t_fir, "T", \%t_hash);
+ if (!$rc) {
+ my $reason = !defined($rc) ? "no header" : "duplicate";
+ my $t_size = $t_fir->{$m}->{"RFC822.SIZE"} || 0;
+ print "+ Skipping msg #$m:$t_size in 'to' folder $t_fold ($reason so we ignore this message)\n";
+ #$mess_size_total_skipped += $msize;
+ #$mess_skipped += 1;
+ }
}
$debug and print "Time headers: ", timenext(), " s\n";
@@ -1580,7 +1589,7 @@
my $flags_f = $f_hash{$m_id}{'F'} || "";
# RFC 2060: This flag can not be altered by any client
- $flags_f =~ s@\\Recent@@gi;
+ $flags_f =~ s@\\Recent\s?@@gi;
$flags_f = flags_regex($flags_f) if @regexflag;
my $new_id;
@@ -1646,14 +1655,24 @@
my $flags_f = $f_hash{$m_id}{'F'} || "";
my $flags_t = $t_hash{$m_id}{'F'} || "";
- # No flag \Recent here, no ?
+ # RFC 2060: This flag can not be altered by any client
+ $flags_f =~ s@\\Recent\s?@@gi;
$flags_f = flags_regex($flags_f) if @regexflag;
- $debug and print "Setting flags from($flags_f) to($flags_t)\n";
-
- # This add or change flags but no flag are removed with this
- $to->store($t_msg, "+FLAGS.SILENT ($flags_f)" )
- if (!$dry and $flags_f ne $flags_t);
+ # compare flags - add missing flags
+ my @ff = split(' ', $flags_f );
+ my %ft = map { $_ => 1 } split(' ', $flags_t );
+ my @flags_a = map { exists $ft{$_} ? () : $_ } @ff;
+
+ $debug and print "Setting flags(@flags_a) ffrom($flags_f) fto($flags_t) on msg #$t_msg in $t_fold\n";
+
+ # This adds or changes flags but no flag are removed with this
+ if (!$dry and @flags_a and !$to->store($t_msg, "+FLAGS.SILENT (@flags_a)") ) {
+ warn "Could not add flags '@flags_a' flagf '$flags_f'",
+ " flagt '$flags_t' on msg #$t_msg in $t_fold: ",
+ $to->LastError, "\n";
+ #$error++;
+ }
last FOLDER if $to->IsUnconnected();
$debug and do {
@@ -1979,7 +1998,7 @@
#$headstr = $imap->message_string($m_uid);
print "no header so we ignore this message\n";
- return;
+ return undef;
}
my $size = $s_fir->{$m_uid}->{"RFC822.SIZE"};
my $flags = $s_fir->{$m_uid}->{"FLAGS"};
@@ -1994,6 +2013,8 @@
else {
$key = "$m_md5:$size";
}
+ # 0 return code is used to identify duplicate message hash
+ return 0 if exists $s_hash->{"$key"};
$s_hash->{"$key"}{'5'} = $m_md5;
$s_hash->{"$key"}{'s'} = $size;
$s_hash->{"$key"}{'D'} = $idate;

98
patches/imapsync.1.284.patch Normal file
View file

@ -0,0 +1,98 @@
--- imapsync.ORIG 2009-06-30 15:39:54.156250000 -0400
+++ imapsync 2009-06-30 15:55:06.703125000 -0400
@@ -1237,7 +1237,7 @@
my ($t_fold);
my ($x_fold) = @_;
# first we remove the prefix
- $x_fold =~ s/^$f_prefix//;
+ $x_fold =~ s/^\Q$f_prefix\E//;
$debug and print "removed source prefix: [$x_fold]\n";
$t_fold = separator_invert($x_fold,$f_sep, $t_sep);
$debug and print "inverted separators: [$t_fold]\n";
@@ -1406,9 +1406,11 @@
last FOLDER if $from->IsUnconnected();
foreach my $m (@f_msgs) {
- unless (parse_header_msg1($from, $m, $f_heads, $f_fir, "F", \%f_hash)) {
+ my $rc = parse_header_msg1($from, $m, $f_heads, $f_fir, "F", \%f_hash);
+ if (!$rc) {
+ my $reason = !defined($rc) ? "no header" : "duplicate";
my $f_size = $f_fir->{$m}->{"RFC822.SIZE"} || 0;
- print "+ Skipping msg #$m:$f_size in folder $f_fold (no header so we ignore this message)\n";
+ print "+ Skipping msg #$m:$f_size in folder $f_fold ($reason so we ignore this message)\n";
$mess_size_total_skipped += $f_size;
$mess_skipped += 1;
}
@@ -1427,7 +1429,14 @@
$debug and print "Time fir: ", timenext(), " s\n";
last FOLDER if $to->IsUnconnected();
foreach my $m (@t_msgs) {
- parse_header_msg1($to, $m, $t_heads, $t_fir, "T", \%t_hash);
+ my $rc = parse_header_msg1($to, $m, $t_heads, $t_fir, "T", \%t_hash);
+ if (!$rc) {
+ my $reason = !defined($rc) ? "no header" : "duplicate";
+ my $t_size = $t_fir->{$m}->{"RFC822.SIZE"} || 0;
+ print "+ Skipping msg #$m:$t_size in 'to' folder $t_fold ($reason so we ignore this message)\n";
+ #$mess_size_total_skipped += $msize;
+ #$mess_skipped += 1;
+ }
}
$debug and print "Time headers: ", timenext(), " s\n";
@@ -1572,7 +1581,7 @@
my $flags_f = $f_hash{$m_id}{'F'} || "";
# RFC 2060: This flag can not be altered by any client
- $flags_f =~ s@\\Recent@@gi;
+ $flags_f =~ s@\\Recent\s?@@gi;
$flags_f = flags_regex($flags_f) if @regexflag;
my $new_id;
@@ -1638,14 +1647,24 @@
my $flags_f = $f_hash{$m_id}{'F'} || "";
my $flags_t = $t_hash{$m_id}{'F'} || "";
- # No flag \Recent here, no ?
+ # RFC 2060: This flag can not be altered by any client
+ $flags_f =~ s@\\Recent\s?@@gi;
$flags_f = flags_regex($flags_f) if @regexflag;
- $debug and print "Setting flags from($flags_f) to($flags_t)\n";
+ # compare flags - add missing flags
+ my @ff = split(' ', $flags_f );
+ my %ft = map { $_ => 1 } split(' ', $flags_t );
+ my @flags_a = map { exists $ft{$_} ? () : $_ } @ff;
- # This add or change flags but no flag are removed with this
- $to->store($t_msg, "+FLAGS.SILENT ($flags_f)" )
- if (!$dry and $flags_f ne $flags_t);
+ $debug and print "Setting flags(@flags_a) ffrom($flags_f) fto($flags_t) on msg #$t_msg in $t_fold\n";
+
+ # This adds or changes flags but no flag are removed with this
+ if (!$dry and @flags_a and !$to->store($t_msg, "+FLAGS.SILENT (@flags_a)") ) {
+ warn "Could not add flags '@flags_a' flagf '$flags_f'",
+ " flagt '$flags_t' on msg #$t_msg in $t_fold: ",
+ $to->LastError, "\n";
+ #$error++;
+ }
last FOLDER if $to->IsUnconnected();
$debug and do {
@@ -1971,7 +1990,7 @@
#$headstr = $imap->message_string($m_uid);
print "no header so we ignore this message\n";
- return;
+ return undef;
}
my $size = $s_fir->{$m_uid}->{"RFC822.SIZE"};
my $flags = $s_fir->{$m_uid}->{"FLAGS"};
@@ -1986,6 +2005,8 @@
else {
$key = "$m_md5:$size";
}
+ # 0 return code is used to identify duplicate message hash
+ return 0 if exists $s_hash->{"$key"};
$s_hash->{"$key"}{'5'} = $m_md5;
$s_hash->{"$key"}{'s'} = $size;
$s_hash->{"$key"}{'D'} = $idate;

40
tests.sh
View file

@ -1,6 +1,6 @@
#!/bin/sh
# $Id: tests.sh,v 1.91 2009/06/30 03:16:31 gilles Exp gilles $
# $Id: tests.sh,v 1.92 2009/07/03 01:00:42 gilles Exp gilles $
# Example:
# CMD_PERL='perl -I./Mail-IMAPClient-3.14/lib' sh -x tests.sh
@ -91,6 +91,12 @@ sendtestmessage() {
}
can_send() {
test X`hostname` = X"plume" && return 0;
test X`hostname` = X"vadrouille" && return 0;
return 1
}
zzzz() {
$CMD_PERL -V
@ -136,7 +142,7 @@ first_sync() {
locallocal() {
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -246,7 +252,7 @@ ll_prefix12() {
ll_internaldate() {
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -264,7 +270,7 @@ ll_internaldate() {
ll_idatefromheader() {
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -362,7 +368,7 @@ ll_noauthmd5()
ll_maxage()
{
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -382,7 +388,7 @@ ll_maxage()
ll_maxsize()
{
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -400,7 +406,7 @@ ll_maxsize()
ll_skipsize()
{
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -417,7 +423,7 @@ ll_skipsize()
ll_skipheader()
{
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -436,7 +442,7 @@ ll_skipheader()
ll_include()
{
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -453,7 +459,7 @@ ll_include()
ll_exclude()
{
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -472,7 +478,7 @@ ll_exclude()
ll_regextrans2()
{
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -563,7 +569,7 @@ ll_useheader()
ll_regexmess()
{
if test X`hostname` = X"plume"; then
if can_send; then
rm -f /home/vmail/titi/.yop.yap/cur/*
fi
$CMD_PERL ./imapsync \
@ -577,7 +583,7 @@ ll_regexmess()
--debug \
--allow3xx
if test X`hostname` = X"plume"; then
if can_send; then
file=`ls -t /home/vmail/titi/.yop.yap/cur/* | tail -1`
diff ../../var/imapsync/tests/ll_regexmess/dest_01 $file
#echo 'rm -f /home/vmail/titi/.yop.yap/cur/*'
@ -639,7 +645,7 @@ ssl_justconnect() {
}
ll_ssl() {
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -705,7 +711,7 @@ ll_authmech_CRAMMD5() {
}
ll_delete2() {
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage titi
else
@ -802,7 +808,7 @@ gmail_gmail2() {
allow3xx() {
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage
else
@ -819,7 +825,7 @@ allow3xx() {
archiveopteryx_1() {
if test X`hostname` = X"plume"; then
if can_send; then
#echo3 Here is plume
sendtestmessage je@lupus.aox.org
else