From 54b0fc1d9d0dc4726d61dd48611de4bd05ec2db2 Mon Sep 17 00:00:00 2001 From: Nick Bebout Date: Tue, 14 Jun 2011 06:02:33 -0500 Subject: [PATCH] 1.434 --- ChangeLog | 65 +++++- FAQ | 26 ++- Makefile | 11 +- README | 12 +- TIME | 6 + TODO | 16 +- VERSION | 2 +- VERSION_EXE | 2 +- imapsync | 499 ++++++++++++++++++++++++++++++---------- index.shtml | 35 ++- paypal_return.shtml | 25 +- test2.bat | 11 +- tests.sh | 100 +++++++- tmp/cache/F1/F2/100_200 | 0 tmp/cache/F1/F2/101_201 | 0 tmp/cache/F1/F2/120_220 | 0 tmp/cache/F1/F2/155_255 | 0 tmp/cache/F1/F2/177_277 | 0 18 files changed, 640 insertions(+), 170 deletions(-) delete mode 100644 tmp/cache/F1/F2/100_200 delete mode 100644 tmp/cache/F1/F2/101_201 delete mode 100644 tmp/cache/F1/F2/120_220 delete mode 100644 tmp/cache/F1/F2/155_255 delete mode 100644 tmp/cache/F1/F2/177_277 diff --git a/ChangeLog b/ChangeLog index 0f77648..55c09f4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,17 +1,74 @@ RCS file: RCS/imapsync,v Working file: imapsync -head: 1.422 +head: 1.434 branch: locks: strict - gilles: 1.422 + gilles: 1.434 access list: symbolic names: keyword substitution: kv -total revisions: 422; selected revisions: 422 +total revisions: 434; selected revisions: 434 description: ---------------------------- -revision 1.422 locked by: gilles; +revision 1.434 locked by: gilles; +date: 2011/05/16 07:16:19; author: gilles; state: Exp; lines: +142 -57 +Bugfix. Made --usecache work with --maxage or --maxsize or --min* +---------------------------- +revision 1.433 +date: 2011/05/16 03:07:59; author: gilles; state: Exp; lines: +191 -49 +Improved the way imapsync deals with headers: + - Stopped getting first 2KB of message. Not a good idea. + - If $imap2->parse_headers() fails then take the whole header (instead of body). + - Default is like --useheader Message-Id --useheader Message-ID + - Use header Message-Id and header Date as sig md5 when taking the whole header. + +Better output in debug mode. +---------------------------- +revision 1.432 +date: 2011/05/15 22:06:19; author: gilles; state: Exp; lines: +8 -7 +Options --usecache and --maxsize --minsize can safely be used if --delete is there +---------------------------- +revision 1.431 +date: 2011/05/15 00:47:03; author: gilles; state: Exp; lines: +17 -8 +Added tests of mkpath very long path > 300 char. Win32 fails on them. +---------------------------- +revision 1.430 +date: 2011/05/14 23:13:06; author: gilles; state: Exp; lines: +14 -9 +Bugfix. Added special case for Inbox vs INBOX bug creation ("Couldn't create folder [Inbox] from [INBOX]: 143 NO INBOX already exists!") +---------------------------- +revision 1.429 +date: 2011/05/14 21:24:31; author: gilles; state: Exp; lines: +8 -8 +Adapted regression tests for good_date() when no zone is given. +---------------------------- +revision 1.428 +date: 2011/05/14 02:31:02; author: gilles; state: Exp; lines: +8 -8 +Bugfix. intarnal date needs zone data. Default to +0000. +---------------------------- +revision 1.427 +date: 2011/05/14 01:48:07; author: gilles; state: Exp; lines: +15 -13 +Bugfix. Starttls() only for 2.2.9 +---------------------------- +revision 1.426 +date: 2011/05/14 01:14:16; author: gilles; state: Exp; lines: +8 -8 +Fix. Removed a debug print always printed. +---------------------------- +revision 1.425 +date: 2011/05/14 00:33:46; author: gilles; state: Exp; lines: +16 -15 +Bugfix. Changed the way imapsync knows whether a folder exists or not. Exchange might be happy and stop deconnecting for this reason. +---------------------------- +revision 1.424 +date: 2011/05/14 00:12:38; author: gilles; state: Exp; lines: +20 -9 +Added a warning and die if --usecache and one of --maxsize--minsize --maxage --minage is used. +---------------------------- +revision 1.423 +date: 2011/05/13 22:26:01; author: gilles; state: Exp; lines: +23 -12 +Bugfix. Reconnections are well done in tls mode now. +Zimbra 5.0.24_GA_3356.RHEL4 [host1] +Exchange 2010 SP1 RU2 [host2] +Added --debugsleep to have to play will kill and reconnections. +---------------------------- +revision 1.422 date: 2011/05/08 17:21:38; author: gilles; state: Exp; lines: +17 -12 Added --debugLIST to track messages list uid or number only. Bugfix: a lack of variable initialisation caused to fetch no existing messages. diff --git a/FAQ b/FAQ index 10ffef0..16d7d52 100644 --- a/FAQ +++ b/FAQ @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: FAQ,v 1.85 2011/02/28 16:02:17 gilles Exp gilles $ +# $Id: FAQ,v 1.86 2011/05/16 16:43:12 gilles Exp gilles $ +------------------+ | FAQ for imapsync | @@ -533,14 +533,21 @@ R. Do not use the --folder option. Q. How to migrate from or to Exchange 2007/2010 with an admin/authuser account? -R. The trick comes from Michele Marcionelli: +R. The tricks comes from Michele Marcionelli and Benjamin Priestman: This doesn't work: -imapsync ... --user2 user2 --authuser2 admin2 --password2 adminpassword2 ... + imapsync ... --user2 user2 --authuser2 admin2 --password2 adminpassword2 ... -This works: -imapsync ... --user2 'domain\admin2\user2' --password2 adminpassword2 ... +This might works: + imapsync ... --user2 'domain\admin2\user2' --password2 adminpassword2 ... +or + imapsync ... --user2 'admin2@domain\user2' --password2 adminpassword2 ... +where "domain" is set be the user's UPN in Active Directory +or the NETBIOS or DNS name of the domain. + +The exact format might vary depending on local configuration and you +should experiment with the different formats. ====================================================================== Q. How to migrate from uw-imap with an admin/authuser account? @@ -845,6 +852,15 @@ format issues. And now it works fine. (Thanks to Hansjoerg.Maurer) Server specific issues and solutions ======================================================================= +======================================================================= +Q. From Zimbra to XXX + +imapsync ... \ + --exclude "Conversation Action Settings" \ + --exclude "Quick Step Settings" \ + --exclude "News Feed" + +======================================================================= Q. From or to HMailServer version 4.4.1. R. You have to add prefix and separator manually because 4.4.1 doesn't diff --git a/Makefile b/Makefile index 2d47efa..e2801a6 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -# $Id: Makefile,v 1.72 2011/05/09 00:11:00 gilles Exp gilles $ +# $Id: Makefile,v 1.74 2011/05/16 17:25:22 gilles Exp gilles $ .PHONY: help usage all @@ -10,6 +10,7 @@ usage: @echo "make install # as root" @echo "make testf # run tests" @echo "make testv # run tests verbosely" + @echo "make test_quick # few tests verbosely" @echo "make test3xx # run tests with (last) Mail-IMAPClient-3.xy" @echo "make test229 # run tests with Mail-IMAPClient-2.2.9" @echo "make tests_win32 # run tests on win32" @@ -84,13 +85,13 @@ cidone: .PHONY: test tests testp testf test3xx -test_quick : test_quick_229 test_quick_3xx +test_quick : test_quick_3xx test_quick_229 test_quick_229: imapsync tests.sh - CMD_PERL='perl -I./Mail-IMAPClient-2.2.9' /usr/bin/time sh tests.sh locallocal 1>/dev/null + CMD_PERL='perl -I./Mail-IMAPClient-2.2.9' /usr/bin/time sh -x tests.sh locallocal test_quick_3xx: imapsync tests.sh - CMD_PERL='perl -I./Mail-IMAPClient-3.28/lib' /usr/bin/time sh tests.sh locallocal 1>/dev/null + CMD_PERL='perl -I./Mail-IMAPClient-3.28/lib' /usr/bin/time sh -x tests.sh locallocal testv: sh -x tests.sh @@ -192,7 +193,7 @@ lfo: cidone niouze_lfo upload_lfo dist: cidone test clean all INSTALL tarball -tarball: cidone all imapsync.exe +tarball: cidone all echo making tarball $(DIST_FILE) mkdir -p dist mkdir -p ../prepa_dist/$(DIST_NAME) diff --git a/README b/README index 8fcd6c0..04153c8 100644 --- a/README +++ b/README @@ -3,7 +3,7 @@ NAME Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 36 different IMAP server softwares supported with success. - $Revision: 1.422 $ + $Revision: 1.434 $ SYNOPSIS To synchronise imap account "foo" on "imap.truc.org" to imap account @@ -336,8 +336,9 @@ IMAP SERVERS - Mercury 4.1 (Windows server 2000 platform) - Microsoft Exchange Server 5.5, 6.0.6249.0[host1], 6.0.6487.0[host1], 6.5.7638.1 [host2], 6.5 [host1], Exchange 2007 SP1 (with Update Rollup 2), - Exchange2007-EP-SP2, - Exchange 2010 RTM (Release to Manufacturing) [host2] + Exchange2007-EP-SP2, + Exchange 2010 RTM (Release to Manufacturing) [host2], + Exchange 2010 SP1 RU2[host2], - Mirapoint - Netscape Mail Server 3.6 (Wintel !) - Netscape Messaging Server 4.15 Patch 7 @@ -357,7 +358,8 @@ IMAP SERVERS (http://www.washington.edu/imap/) - UW - QMail v2.1 - Imap part of TCP/IP suite of VMS 7.3.2 - - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5, 6.x + - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, + Zimbra 5.0.24_GA_3356.RHEL4 [host1], 5.5, 6.x Please report to the author any success or bad story with imapsync and do not forget to mention the IMAP server software names and version on @@ -422,5 +424,5 @@ SIMILAR SOFTWARES Feedback (good or bad) will often be welcome. - $Id: imapsync,v 1.422 2011/05/08 17:21:38 gilles Exp gilles $ + $Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ diff --git a/TIME b/TIME index 0763dd5..3023723 100644 --- a/TIME +++ b/TIME @@ -1,3 +1,9 @@ +540 (1.434) (1.433) (1.432) +180 Tests of mkpath very long path > 300 char. Win32 fails on them. (1.431) + Added special case for Inbox vs INBOX bug creation. (1.430) + Adapted regression tests for good_date() when no zone is given. (1.429) + 60 Reading, replying and fixing from Benjamin Priestman feedback. + 90 Various email help. 300 Release 1.417. Some numbers section. INSTALL file. 60 Bugfix. --maxsize --minsize now work with --useuid + flag sync of already transfered messages now take care of --maxsize --minsize options. 120 Exit on --delete --delete2. --expunge1 same as --expunge. --delete2 implies --expunge2. diff --git a/TODO b/TODO index 0a1b661..7c4518c 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: TODO,v 1.96 2011/04/26 10:48:03 gilles Exp gilles $ +# $Id: TODO,v 1.97 2011/05/16 16:39:38 gilles Exp gilles $ TODO file for imapsync ---------------------- @@ -23,10 +23,8 @@ Evaluate http://www.rackspace.com/apps/email_hosting/migrations http://www.yippiemove.com/ -Fix Exchange 2010 SP1 issue with --foldersizes when -host2 folders don't exist. $imap->exists calls STATUS. -Is it RFC compliant or an Exchange bug? -Exchange quit after 10 errors. +Fix long path over than 256 character on Win32. +Think about Digest::SHA or Digest::SHA::PurePerl. Fix "\Forwarded" flag bug in courier. Does \lalala can be forbidden (courier does a @@ -35,7 +33,6 @@ with * OK [PERMANENTFLAGS (\* \Draft \Answered \Flagged \Deleted \Seen)] Limited - Suggestion: it's very difficult to track down messages which are behaving funny during the sync. It would be great - and presumably easy to code - to have an option to have imapsync display e.g. the subject of an @@ -159,6 +156,13 @@ http://asg.web.cmu.edu/cyrus/download/imapd/altnamespace.html =========================================================================== +DONE. Make --usecache works with --maxage --maxsize etc. + +DONE. Fix Exchange 2010 SP1 issue with --foldersizes when +host2 folders don't exist. $imap->exists calls STATUS. +Is it RFC compliant or an Exchange bug? +Exchange quit after 10 errors. + DONE. Make --delete2 works with --useuid DONE. write a comment to http://blog.migrationwiz.com/2010/12/09/imapsync-vs-migrationwiz/ diff --git a/VERSION b/VERSION index fefe318..d18240f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.422 +1.434 diff --git a/VERSION_EXE b/VERSION_EXE index 9206104..c31093c 100644 --- a/VERSION_EXE +++ b/VERSION_EXE @@ -1 +1 @@ -1.422 +1.434 diff --git a/imapsync b/imapsync index e6ba579..9055ab3 100755 --- a/imapsync +++ b/imapsync @@ -20,7 +20,7 @@ Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 36 different IMAP server softwares supported with success. -$Revision: 1.422 $ +$Revision: 1.434 $ =head1 SYNOPSIS @@ -384,8 +384,9 @@ Success stories reported with the following 41 imap servers - Mercury 4.1 (Windows server 2000 platform) - Microsoft Exchange Server 5.5, 6.0.6249.0[host1], 6.0.6487.0[host1], 6.5.7638.1 [host2], 6.5 [host1], Exchange 2007 SP1 (with Update Rollup 2), - Exchange2007-EP-SP2, - Exchange 2010 RTM (Release to Manufacturing) [host2] + Exchange2007-EP-SP2, + Exchange 2010 RTM (Release to Manufacturing) [host2], + Exchange 2010 SP1 RU2[host2], - Mirapoint - Netscape Mail Server 3.6 (Wintel !) - Netscape Messaging Server 4.15 Patch 7 @@ -405,7 +406,8 @@ Success stories reported with the following 41 imap servers (http://www.washington.edu/imap/) - UW - QMail v2.1 - Imap part of TCP/IP suite of VMS 7.3.2 - - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5, 6.x + - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, + Zimbra 5.0.24_GA_3356.RHEL4 [host1], 5.5, 6.x Please report to the author any success or bad story with imapsync and do not forget to mention the IMAP server @@ -496,7 +498,7 @@ Entries for imapsync: Feedback (good or bad) will often be welcome. -$Id: imapsync,v 1.422 2011/05/08 17:21:38 gilles Exp gilles $ +$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ =cut @@ -541,7 +543,7 @@ use constant { my( $rcs, $pidfile, $debug, $debugimap, $debugimap1, $debugimap2, $debugcontent, $debugflags, - $debugLIST, + $debugLIST, $debugsleep, $nb_errors, $host1, $host2, $port1, $port2, $user1, $user2, $domain1, $domain2, @@ -601,14 +603,14 @@ my( $modules_version, $delete2folders, $delete2foldersonly, $delete2foldersbutnot, $usecache, $debugcache, - $takebody, %h1_msgs_copy_by_uid, $useuid, $h2_uidguess, + $wholeheaderifneeded, %h1_msgs_copy_by_uid, $useuid, $h2_uidguess, ); # main program # global variables initialisation -$rcs = '$Id: imapsync,v 1.422 2011/05/08 17:21:38 gilles Exp gilles $ '; +$rcs = '$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ '; $total_bytes_transferred = 0; $total_bytes_skipped = 0; @@ -650,7 +652,7 @@ $pidfile ||= $tmpdir . '/imapsync.pid'; # allow Mail::IMAPClient 3.0.xx by default $allow3xx = defined($allow3xx) ? $allow3xx : 1; -$takebody = defined( $takebody ) ? $takebody : 1; +$wholeheaderifneeded = defined( $wholeheaderifneeded ) ? $wholeheaderifneeded : 1; # turn on RFC standard flags correction like \SEEN -> \Seen $flagsCase = defined( $flagsCase ) ? $flagsCase : 1 ; @@ -740,14 +742,12 @@ if ( $delete and $delete2 ) { exit_clean( 0 ) ; } - - -if($idatefromheader) { +if ($idatefromheader) { print "Turned ON idatefromheader, ", "will set the internal dates on host2 from the 'Date:' header line.\n"; $syncinternaldates = 0; - } + if ($syncinternaldates) { print "Turned ON syncinternaldates, ", "will set the internal dates (arrival dates) on host2 same as host1.\n"; @@ -755,19 +755,19 @@ if ($syncinternaldates) { print "Turned OFF syncinternaldates\n"; } -if(defined($authmd5) and ($authmd5)) { +if (defined($authmd5) and ($authmd5)) { $authmd51 = 1 ; $authmd52 = 1 ; } -if(defined($authmd51) and ($authmd51)) { +if (defined($authmd51) and ($authmd51)) { $authmech1 ||= 'CRAM-MD5'; } else{ $authmech1 ||= $authuser1 ? 'PLAIN' : 'LOGIN'; } -if(defined($authmd52) and ($authmd52)) { +if (defined($authmd52) and ($authmd52)) { $authmech2 ||= 'CRAM-MD5'; } else{ @@ -800,7 +800,19 @@ $fastio2 = (defined($fastio2)) ? $fastio2 : 0; $reconnectretry1 = (defined($reconnectretry1)) ? $reconnectretry1 : 3; $reconnectretry2 = (defined($reconnectretry2)) ? $reconnectretry2 : 3; -@useheader = ("Message-Id") unless (@useheader); +@useheader = ( "Message-Id", "Message-ID" ) unless ( @useheader ) ; + +my %useheader ; + +# Make a hash %useheader of each --useheader 'key' in uppercase +@useheader{ map( { uc( $_ ) } @useheader ) } = ( ) ; + +my %useheaderclassic ; +@useheaderclassic{ qw(MESSAGE-ID DATE) } = ( ) ; + +#require Data::Dumper ; +#print Data::Dumper->Dump( [ \%useheader ] ) ; + print "Host1: imap server [$host1] port [$port1] user [$user1]\n"; print "Host2: imap server [$host2] port [$port2] user [$user2]\n"; @@ -817,6 +829,18 @@ $password2 || $passfile2 || do { $password2 = (defined($passfile2)) ? firstline ($passfile2) : $password2; +if ( ! ( 1 + or ( $maxsize + or $minsize + or $maxage + or $minage ) + and $usecache + and ! $delete ) ) { + die_clean( +"Problem --usecache can not be used safely with options --maxsize--minsize --maxage --minage +Use --nousecache or suppress the --max* --min* options\n" ) ; +} + my $imap1 = (); my $imap2 = (); @@ -1032,15 +1056,21 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { next FOLDER if ($justfolders); - my @h1_msgs = select_msgs($imap1); + my $h1_msgs_all_hash_ref = { } ; + my @h1_msgs = select_msgs( $imap1, $h1_msgs_all_hash_ref ); - ( $debug or $debugLIST ) and print "LIST Host1: ", scalar(@h1_msgs), " messages [@h1_msgs]\n"; + ( $debug or $debugLIST ) and print "Host1 LIST: ", scalar( @h1_msgs ), " messages [@h1_msgs]\n" ; + #( $debug or $debugLIST ) and print "Host1 LIST: ", scalar( @$h1_msgs_all_ref ), " messages [@$h1_msgs_all_ref]\n" ; # internal dates on host2 are after the ones on host1 # normally... - my @h2_msgs = select_msgs($imap2); + + my $h2_msgs_all_hash_ref = { } ; + my @h2_msgs = select_msgs( $imap2, $h2_msgs_all_hash_ref ) ; - ( $debug or $debugLIST ) and print "LIST Host2: ", scalar(@h2_msgs), " messages [@h2_msgs]\n"; - + ( $debug or $debugLIST ) and print "Host2 LIST: ", scalar(@h2_msgs), " messages [@h2_msgs]\n"; + #( $debug or $debugLIST ) and print "Host2 LIST ALL: ", scalar( keys %$h2_msgs_all_hash_ref ), " messages [", + # join( ' ', keys( %$h2_msgs_all_hash_ref ) ), "]\n" ; + my $cache_base = "$tmpdir/imapsync_cache/$host1/$user1/$host2/$user2"; my $cache_dir = cache_folder( $cache_base, $h1_fold, $h2_fold ); my ( $cache_1_2_ref, $cache_2_1_ref ) = ( {}, {} ); @@ -1048,7 +1078,8 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { if ( $usecache ) { print "cache directory: $cache_dir\n" ; mkpath( "$cache_dir" ) ; - ( $cache_1_2_ref, $cache_2_1_ref ) = get_cache($cache_dir, \@h1_msgs, \@h2_msgs) if ($usecache) ; + ( $cache_1_2_ref, $cache_2_1_ref ) + = get_cache( $cache_dir, \@h1_msgs, \@h2_msgs, $h1_msgs_all_hash_ref, $h2_msgs_all_hash_ref ) ; print "CACHE h1 h2: ", scalar( keys %$cache_1_2_ref ), " files\n" ; $debug and print '[', map ( { "$_->$cache_1_2_ref->{$_} " } keys %$cache_1_2_ref ), " ]\n"; @@ -1058,16 +1089,16 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { my %h1_hash = (); my %h2_hash = (); - my ( %h1_msgs_all, %h2_msgs_all ) ; - @h1_msgs_all{ @h1_msgs } = (); - @h2_msgs_all{ @h2_msgs } = (); + my ( %h1_msgs, %h2_msgs ) ; + @h1_msgs{ @h1_msgs } = (); + @h2_msgs{ @h2_msgs } = (); my @h1_msgs_in_cache = sort { $a <=> $b } keys %$cache_1_2_ref ; my @h2_msgs_in_cache = keys %$cache_2_1_ref ; my ( %h1_msgs_no_cache, %h2_msgs_no_cache ) ; - %h1_msgs_no_cache = %h1_msgs_all ; - %h2_msgs_no_cache = %h2_msgs_all ; + %h1_msgs_no_cache = %h1_msgs ; + %h2_msgs_no_cache = %h2_msgs ; delete @h1_msgs_no_cache{ @h1_msgs_in_cache } ; delete @h2_msgs_no_cache{ @h2_msgs_in_cache } ; @@ -1088,16 +1119,18 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { #print "delete2: @h2_msgs_delete2_no_cache\n"; } - $debug and print "Host1 folder [$h1_fold] parsing headers\n"; + $debug and print "Host1 parsing headers of folder [$h1_fold]\n"; my ($h1_heads_ref, $h1_fir_ref) = ({}, {}); $h1_heads_ref = $imap1->parse_headers([@h1_msgs_no_cache], @useheader) if (@h1_msgs_no_cache); - $debug and print "Time headers: ", timenext(), " s\n"; + $debug and print "Host1 parsing headers of folder [$h1_fold] took ", timenext(), " s\n"; @$h1_fir_ref{@h1_msgs} = (undef); + + $debug and print "Host1 getting flags idate and sizes of folder [$h1_fold]\n" ; $h1_fir_ref = $imap1->fetch_hash_2("FLAGS", "INTERNALDATE", "RFC822.SIZE", $h1_fir_ref) if (@h1_msgs); - $debug and print "Time fir: ", timenext(), " s\n"; + $debug and print "Host1 getting flags idate and sizes of folder [$h1_fold] took ", timenext(), " s\n"; unless ($h1_fir_ref) { warn "Host1 folder $h1_fold: Could not fetch_hash_2 ", @@ -1108,10 +1141,10 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { my @h1_msgs_duplicate; foreach my $m (@h1_msgs_no_cache) { - my $rc = parse_header_msg($imap1, $m, $h1_heads_ref, $h1_fir_ref, "F", \%h1_hash); + my $rc = parse_header_msg($imap1, $m, $h1_heads_ref, $h1_fir_ref, 'Host1', \%h1_hash); if (! defined($rc)) { my $h1_size = $h1_fir_ref->{$m}->{"RFC822.SIZE"} || 0; - print "+ Skipping msg #$m:$h1_size on host1 folder $h1_fold (no header so we ignore this message)\n"; + print "Host1 $h1_fold/$m size $h1_size ignored (no header so we ignore this message)\n"; $total_bytes_skipped += $h1_size; $nb_msg_skipped += 1; $h1_nb_msg_noheader +=1; @@ -1125,25 +1158,27 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { $h1_nb_msg_duplicate += 1; } } - $debug and print "Time parsing headers on host1: ", timenext(), " s\n"; - - $debug and print "Host2 folder [$h2_fold] parsing headers\n"; - + $debug and print "Host1 whole time parsing headers took ", timenext(), " s\n"; + $debug and print "\n"; + + $debug and print "Host2 parsing headers of folder [$h2_fold]\n"; + my ($h2_heads_ref, $h2_fir_ref) = ( {}, {} ); $h2_heads_ref = $imap2->parse_headers([@h2_msgs_no_cache], @useheader) if (@h2_msgs_no_cache); - $debug and print "Time headers: ", timenext(), " s\n"; + $debug and print "Host2 parsing headers of folder [$h2_fold] took ", timenext(), " s\n" ; + $debug and print "Host2 getting flags idate and sizes of folder [$h2_fold]\n" ; @$h2_fir_ref{@h2_msgs} = ( ); # fetch_hash_2 can select by uid with last arg as ref $h2_fir_ref = $imap2->fetch_hash_2("FLAGS", "INTERNALDATE", "RFC822.SIZE", $h2_fir_ref) if (@h2_msgs); - $debug and print "Time fir: ", timenext(), " s\n"; + $debug and print "Host2 getting flags idate and sizes of folder [$h2_fold] took ", timenext(), " s\n" ; my @h2_msgs_duplicate; foreach my $m (@h2_msgs_no_cache) { - my $rc = parse_header_msg($imap2, $m, $h2_heads_ref, $h2_fir_ref, "T", \%h2_hash); + my $rc = parse_header_msg($imap2, $m, $h2_heads_ref, $h2_fir_ref, 'Host2', \%h2_hash); my $h2_size = $h2_fir_ref->{$m}->{"RFC822.SIZE"} || 0; if (! defined($rc)) { - print "+ Skipping msg #$m:$h2_size in host2 folder $h2_fold (no header so we ignore this message)\n"; + print "Host2 $h2_fold/$m size $h2_size ignored (no header so we ignore this message)\n"; $h2_nb_msg_noheader += 1 ; } elsif(0 == $rc) { # duplicate @@ -1152,7 +1187,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { push(@h2_msgs_duplicate, $m); } } - $debug and print "Time parsing headers on host2: ", timenext(), " s\n"; + $debug and print "Host2 whole time parsing headers took ", timenext(), " s\n"; $debug and print "++++ Verifying [$h1_fold] -> [$h2_fold]\n"; # messages in host1 that are not in host2 @@ -1227,7 +1262,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { else{ # already on host2 my $h2_msg = $h2_hash{$m_id}{'m'} ; - $debug and print "msg $h1_fold/$h1_msg equals $h2_fold/$h2_msg\n" ; + $debug and print "msg $h1_fold/$h1_msg equals $h2_fold/$h2_msg\n" ; $total_bytes_skipped += $h1_size ; $nb_msg_skipped += 1 ; $debugcache and print "touch $cache_dir/${h1_msg}_$h2_msg\n" if ( $usecache ) ; @@ -1242,9 +1277,9 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { # Good my $h2_size = $h2_hash{$m_id}{'s'}; $debug and print - "msg $h1_fold/$h1_msg sizes $h1_size <> $h2_size $h2_fold/$h2_msg\n"; + "msg $h1_fold/$h1_msg sizes $h1_size <> $h2_size $h2_fold/$h2_msg\n"; if( $delete ) { - print "msg $h1_fold/$h1_msg deleted on host1\n"; + print "msg $h1_fold/$h1_msg deleted on host1\n"; unless( $dry ) { $imap1->delete_message( $h1_msg ); $h1_nb_msg_deleted += 1; @@ -1256,7 +1291,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { # END MESS: loop MESS_IN_CACHE: foreach my $h1_msg ( @h1_msgs_in_cache ) { my $h2_msg = $cache_1_2_ref->{ $h1_msg } ; - $debugcache and print "cache messages update $h1_msg->$h2_msg\n"; + $debugcache and print "cache messages update flags $h1_msg->$h2_msg\n"; sync_flags( $h1_fold, $h1_msg, $h2_fold, $h2_msg, $permanentflags2, $h1_fir_ref, $h2_fir_ref ) ; my $h1_size = $h1_fir_ref->{ $h1_msg }->{ 'RFC822.SIZE' } ; $total_bytes_skipped += $h1_size; @@ -1297,7 +1332,7 @@ sub size_filtered_flag { sub sync_flags { my ( $h1_fold, $h1_msg, $h2_fold, $h2_msg, $permanentflags2, $h1_fir_ref, $h2_fir_ref ) = @_ ; - $debug and print "sync flags $h1_fold/$h1_msg->$h2_fold/$h2_msg\n"; + $debug and print "flags $h1_fold/$h1_msg -> $h2_fold/$h2_msg\n"; my $h1_size = $h1_fir_ref->{$h1_msg}->{"RFC822.SIZE"} ; return() if size_filtered_flag( $h1_size ) ; @@ -1498,6 +1533,7 @@ sub myconnect { if ($self->Tls) { starttls($self); + $self->Starttls( 1 ) ; } $self->Ignoresizeerrors($allowsizemismatch); @@ -1727,8 +1763,8 @@ sub banner_imapsync { my @argv_copy = @_; my $banner_imapsync = join("", '$RCSfile: imapsync,v $ ', - '$Revision: 1.422 $ ', - '$Date: 2011/05/08 17:21:38 $ ', + '$Revision: 1.434 $ ', + '$Date: 2011/05/16 07:16:19 $ ', "\n",localhost_info(), "\n", "Command line used:\n", "$0 ", command_line_nopassword(@argv_copy), "\n", @@ -1799,17 +1835,16 @@ sub select_folder { sub create_folder { my( $imap2, $h2_fold, $h1_fold ) = @_ ; - if ( $imap2->exists( $h2_fold ) ) { - print "Folder $h2_fold already exists on host2.\n"; - return( 1 ) ; - }else{ - print "Folder $h2_fold does not exist on host2.\n"; - } - print "Creating folder [$h2_fold] on host2.\n"; + print "Creating folder [$h2_fold] on host2\n"; + if ( ( 'INBOX' eq uc( $h2_fold) ) + and ( $imap2->exists( $h2_fold ) ) ) { + print "Folder [$h2_fold] already exists\n" ; + return( 1 ) ; + } if ( ! $dry ){ if ( ! $imap2->create($h2_fold)){ - warn "Couldn't create folder [$h2_fold] from [$h1_fold]: ", - $imap2->LastError,"\n"; + warn( "Couldn't create folder [$h2_fold] from [$h1_fold]: ", + $imap2->LastError(), "\n" ); $nb_errors++; return(0); }else{ @@ -2123,10 +2158,15 @@ sub foldersizes { my $stot = 0; my $smess = 0; printf("$side folder %-35s", "[$folder]"); - unless($imap->exists($folder)) { - print("does not exist yet\n"); - next; + if ( 'Host2' eq $side and ! exists( $h2_folders_all{ $folder } ) ) { + print(" does not exist yet\n") ; + next ; } + if ( 'Host1' eq $side and ! exists( $h1_folders_all{ $folder } ) ) { + print(" does not exist\n") ; + next ; + } + unless ($imap->examine($folder)) { warn "$side Folder $folder: Could not examine: ", @@ -2410,13 +2450,21 @@ sub tests_ucsecond { } sub select_msgs { - my ($imap) = @_; - my (@msgs,@max,@min,@union,@inter); + my ( $imap, $msgs_all_hash_ref ) = @_ ; + my ( @msgs, @msgs_all, @max, @min, @union, @inter ) ; - unless (defined($maxage) or defined($minage)) { - #@msgs = $imap->search("ALL"); - @msgs = $imap->messages(); - return(@msgs); + + if ( defined( $msgs_all_hash_ref ) + or ( ! defined( $maxage ) and ! defined( $minage ) ) + ) { + @msgs = $imap->messages() ; + if ( defined( $msgs_all_hash_ref ) ) { + @msgs_all = @msgs ; + @{ $msgs_all_hash_ref }{ @msgs_all } = () ; + } + if ( ! defined( $maxage ) and ! defined( $minage ) ) { + return( @msgs ) ; + } } if (defined($maxage)) { @max = $imap->sentsince(time - 86400 * $maxage); @@ -2500,7 +2548,7 @@ sub copy_message { return() if size_filtered( $h1_size, $h1_msg, $h1_fold, $h2_fold ) ; my $string; - #print "SLEEP 5\n" and sleep 5 ; + do { print "SLEEP 5\n" and sleep 5 ; } if ( $debugsleep ) ; print "- msg $h1_fold/$h1_msg S[$h1_size] F[$h1_flags] I[$h1_idate] has RFC822.SIZE null!\n" if ( ! $h1_size ) ; @@ -2678,7 +2726,7 @@ sub tests_cache_map { sub get_cache { $debugcache and print "Entering get_cache\n"; - my ($cache_dir, $h1_msgs_ref, $h2_msgs_ref) = @_; + my ( $cache_dir, $h1_msgs_ref, $h2_msgs_ref, $h1_msgs_all_hash_ref, $h2_msgs_all_hash_ref ) = @_; -d $cache_dir or return( undef ); # exit if cache directory doesn't exist $debugcache and print "cache_dir: $cache_dir\n"; @@ -2690,11 +2738,7 @@ sub get_cache { my( $cache_1_2_ref, $cache_2_1_ref ) = cache_map( \@cache_files, $h1_msgs_ref, $h2_msgs_ref ) ; - clean_cache( \@cache_files, $cache_1_2_ref ) - if ( ! ( defined( $maxsize ) - or defined( $minsize ) - or defined( $maxage ) - or defined( $minage ) ) ); + clean_cache( \@cache_files, $cache_1_2_ref, $h1_msgs_all_hash_ref, $h2_msgs_all_hash_ref ) ; #print "\n", map { "c12 $_ -> $cache_1_2_ref->{ $_ }\n" } keys %$cache_1_2_ref ; #print "\n", map { "c21 $_ -> $cache_2_1_ref->{ $_ }\n" } keys %$cache_2_1_ref ; @@ -2721,16 +2765,19 @@ sub tests_get_cache { tmp/cache/F1/F2/177_777 tmp/cache/F1/F2/155_255 ) ) ; - ok( touch(@test_files_cache), 'get_cache: touch tmp/cache/F1/F2/...' ) ; + ok( touch( @test_files_cache ), 'get_cache: touch tmp/cache/F1/F2/...' ) ; # on cache: 100_200 101_201 142_242 143_243 177_277 177_377 177_777 155_255 # on live: my $msgs_1 = [120, 142, 143, 144, 177 ]; my $msgs_2 = [ 242, 243, 299, 377, 777, 255 ]; + + my $msgs_all_1 = { 120 => '', 142 => '', 143 => '', 144 => '', 177 => '' } ; + my $msgs_all_2 = { 242 => '', 243 => '', 299 => '', 377 => '', 777 => '', 255 => '' } ; my( $c12, $c21 ) ; - ok( ( $c12, $c21 ) = get_cache('tmp/cache/F1/F2', $msgs_1, $msgs_2), 'get_cache: 02' ); + ok( ( $c12, $c21 ) = get_cache( 'tmp/cache/F1/F2', $msgs_1, $msgs_2, $msgs_all_1, $msgs_all_2 ), 'get_cache: 02' ); my $a1 = [ sort { $a <=> $b } keys %$c12 ] ; my $a2 = [ sort { $a <=> $b } keys %$c21 ] ; ok( 0 == compare_lists( [ 142, 143, 177 ], $a1 ), 'get_cache: 03' ); @@ -2740,14 +2787,14 @@ sub tests_get_cache { ok( ! -f 'tmp/cache/F1/F2/100_200', 'get_cache: file removed 100_200'); ok( ! -f 'tmp/cache/F1/F2/101_201', 'get_cache: file removed 101_201'); - # test clean_cache not executed + # test clean_cache executed $maxage = 2 ; ok( touch(@test_files_cache), 'get_cache: touch tmp/cache/F1/F2/...' ) ; - ok( ( $c12, $c21 ) = get_cache('tmp/cache/F1/F2', $msgs_1, $msgs_2), 'get_cache: 02' ); + ok( ( $c12, $c21 ) = get_cache('tmp/cache/F1/F2', $msgs_1, $msgs_2, $msgs_all_1, $msgs_all_2 ), 'get_cache: 02' ); ok( -f 'tmp/cache/F1/F2/142_242', 'get_cache: file kept 142_242'); ok( -f 'tmp/cache/F1/F2/142_242', 'get_cache: file kept 143_243'); - ok( -f 'tmp/cache/F1/F2/100_200', 'get_cache: file NOT removed 100_200'); - ok( -f 'tmp/cache/F1/F2/101_201', 'get_cache: file NOT removed 101_201'); + ok( ! -f 'tmp/cache/F1/F2/100_200', 'get_cache: file NOT removed 100_200'); + ok( ! -f 'tmp/cache/F1/F2/101_201', 'get_cache: file NOT removed 101_201'); # strange files @@ -2774,7 +2821,10 @@ sub tests_get_cache { $msgs_1 = [120, 142, 143, 144, 177 ]; $msgs_2 = [ 242, 243, 299, 377, 777, 255 ]; - ok( ( $c12, $c21 ) = get_cache('tmp/cache/rr\uee', $msgs_1, $msgs_2), 'get_cache: strange path 02' ); + $msgs_all_1 = { 120 => '', 142 => '', 143 => '', 144 => '', 177 => '' } ; + $msgs_all_2 = { 242 => '', 243 => '', 299 => '', 377 => '', 777 => '', 255 => '' } ; + + ok( ( $c12, $c21 ) = get_cache('tmp/cache/rr\uee', $msgs_1, $msgs_2, $msgs_all_1, $msgs_all_2), 'get_cache: strange path 02' ); $a1 = [ sort { $a <=> $b } keys %$c12 ] ; $a2 = [ sort { $a <=> $b } keys %$c21 ] ; ok( 0 == compare_lists( [ 142, 143, 177 ], $a1 ), 'get_cache: strange path 03' ); @@ -2833,8 +2883,7 @@ sub tests_match_a_cache_file { } sub clean_cache { - my $cache_files_ref = shift ; - my $cache_1_2_ref = shift ; + my ( $cache_files_ref, $cache_1_2_ref, $h1_msgs_all_hash_ref, $h2_msgs_all_hash_ref ) = @_ ; $debugcache and print "Entering clean_cache\n"; @@ -2843,10 +2892,13 @@ sub clean_cache { $debugcache and print "$file\n" ; my ( $uid1, $uid2 ) = match_a_cache_file( $file ) ; $debugcache and print "u1: $uid1 u2: $uid2 c12: ", $cache_1_2_ref->{ $uid1 } || '', "\n" ; - if ( ( ! defined( $uid1 ) ) +# or ( ! exists( $cache_1_2_ref->{ $uid1 } ) ) +# or ( ! ( $uid2 == $cache_1_2_ref->{ $uid1 } ) ) + if ( ( ! defined( $uid1 ) ) or ( ! defined( $uid2 ) ) - or ( ! exists( $cache_1_2_ref->{ $uid1 } ) ) - or ( ! ( $uid2 == $cache_1_2_ref->{ $uid1 } ) ) ) { + or ( ! exists( $h1_msgs_all_hash_ref->{ $uid1 } ) ) + or ( ! exists( $h2_msgs_all_hash_ref->{ $uid2 } ) ) + ) { $debugcache and print "remove $file\n" ; unlink( $file ) or warn "$!" ; } @@ -2885,8 +2937,18 @@ sub tests_clean_cache { 142 => 242, 177 => 777, } ; - - ok( clean_cache( \@test_files_cache, $cache ), 'clean_cache: ' ) ; + + my $all_1 = { + 142 => '', + 177 => '', + } ; + + my $all_2 = { + 200 => '', + 242 => '', + 777 => '', + } ; + ok( clean_cache( \@test_files_cache, $cache, $all_1, $all_2 ), 'clean_cache: ' ) ; ok( ! -f 'tmp/cache/G1/G2/100_200', 'clean_cache: 100_200 after' ); ok( -f 'tmp/cache/G1/G2/142_242', 'clean_cache: 142_242 after' ); @@ -2895,7 +2957,61 @@ sub tests_clean_cache { ok( -f 'tmp/cache/G1/G2/177_777', 'clean_cache: 177_777 after' ); ok( ! -f 'tmp/cache/G1/G2/155_255', 'clean_cache: 155_255 after' ); } + +sub tests_clean_cache_2 { + + ok( ( ! -d 'tmp/cache/G1/G2' or rmtree( 'tmp/cache/G1/G2' )), 'clean_cache_2: rmtree tmp/cache/G1/G2' ) ; + ok( mkpath( 'tmp/cache/G1/G2' ), 'clean_cache_2: mkpath tmp/cache/G1/G2' ) ; + my @test_files_cache = ( qw( + tmp/cache/G1/G2/100_200 + tmp/cache/G1/G2/101_201 + tmp/cache/G1/G2/120_220 + tmp/cache/G1/G2/142_242 + tmp/cache/G1/G2/143_243 + tmp/cache/G1/G2/177_277 + tmp/cache/G1/G2/177_377 + tmp/cache/G1/G2/177_777 + tmp/cache/G1/G2/155_255 + ) ) ; + ok( touch(@test_files_cache), 'clean_cache_2: touch tmp/cache/G1/G2/...' ) ; + + ok( -f 'tmp/cache/G1/G2/100_200', 'clean_cache_2: 100_200 before' ); + ok( -f 'tmp/cache/G1/G2/142_242', 'clean_cache_2: 142_242 before' ); + ok( -f 'tmp/cache/G1/G2/177_277', 'clean_cache_2: 177_277 before' ); + ok( -f 'tmp/cache/G1/G2/177_377', 'clean_cache_2: 177_377 before' ); + ok( -f 'tmp/cache/G1/G2/177_777', 'clean_cache_2: 177_777 before' ); + ok( -f 'tmp/cache/G1/G2/155_255', 'clean_cache_2: 155_255 before' ); + + my $cache = { + 142 => 242, + 177 => 777, + } ; + + my $all_1 = { + 100 => '', + 142 => '', + 177 => '', + } ; + + my $all_2 = { + 200 => '', + 242 => '', + 777 => '', + } ; + + + + ok( clean_cache( \@test_files_cache, $cache, $all_1, $all_2 ), 'clean_cache_2: ' ) ; + + ok( -f 'tmp/cache/G1/G2/100_200', 'clean_cache_2: 100_200 after' ); + ok( -f 'tmp/cache/G1/G2/142_242', 'clean_cache_2: 142_242 after' ); + ok( ! -f 'tmp/cache/G1/G2/177_277', 'clean_cache_2: 177_277 after' ); + ok( ! -f 'tmp/cache/G1/G2/177_377', 'clean_cache_2: 177_377 after' ); + ok( -f 'tmp/cache/G1/G2/177_777', 'clean_cache_2: 177_777 after' ); + ok( ! -f 'tmp/cache/G1/G2/155_255', 'clean_cache_2: 155_255 after' ); +} + sub tests_touch { @@ -2908,6 +3024,14 @@ sub tests_touch { } +sub tests_mkpath { + + my $long_path = "123456789/" x 30 ; + ok( (-d "tmp/tests/long/$long_path" or mkpath( "tmp/tests/long/$long_path" )), 'tests_mkpath: mkpath > 300 char' ) ; + ok( (-d "tmp/tests/long/$long_path" and rmtree( "tmp/tests/long/" )), 'tests_mkpath: rmtree > 300 char' ) ; + ok( 1 == 1, 'tests_mkpath: 1 == 1' ) ; +} + sub touch { my @files = @_ ; my @result; @@ -3101,6 +3225,7 @@ sub get_options { "debug!" => \$debug, "debugLIST!" => \$debugLIST, "debugcontent!" => \$debugcontent, + "debugsleep!" => \$debugsleep, "debugflags!" => \$debugflags, "debugimap!" => \$debugimap, "debugimap1!" => \$debugimap1, @@ -3163,7 +3288,7 @@ sub get_options { "timeout=i" => \$timeout, "skipheader=s" => \$skipheader, "useheader=s" => \@useheader, - "takebody!" => \$takebody, + "wholeheaderifneeded!" => \$wholeheaderifneeded, "skipsize!" => \$skipsize, "allowsizemismatch!" => \$allowsizemismatch, "fastio1!" => \$fastio1, @@ -3247,11 +3372,30 @@ sub parse_header_msg { my $head = $s_heads->{$m_uid}; my $headnum = scalar(keys(%$head)); - $debug and print "Head NUM:", $headnum, "\n"; - unless($headnum) { print "Warning: no header used or found for message $m_uid\n"; } + $debug and print "$s uid $m_uid head nb pass one: ", $headnum, "\n"; + my $headstr; + if ( ( ! $headnum ) and ( $wholeheaderifneeded ) ){ + print "$s uid $m_uid no header by classic way so taking whole header\n"; + $imap->fetch($m_uid, "BODY.PEEK[HEADER]"); + my $whole_header = $imap->_transaction_literals; + + #print $whole_header; + $head = decompose_header( $whole_header ) ; + + $headnum = scalar( keys( %$head ) ) ; + $debug and print "$s uid $m_uid head nb pass two: ", $headnum, "\n"; + } + + #require Data::Dumper ; + #print Data::Dumper->Dump( [ $head, \%useheader ] ) ; + foreach my $h (sort keys(%$head)){ + next if ( ! exists( $useheader{ uc( $h ) } ) + and ! exists( $useheader{ 'ALL' } ) + and ! exists( $useheaderclassic{ uc( $h ) } ) + ) ; foreach my $val (sort @{$head->{$h}}) { # no 8-bit data in headers ! $val =~ s/[\x80-\xff]/X/g; @@ -3264,29 +3408,16 @@ sub parse_header_msg { my $H = uc("$h: $val"); # show stuff in debug mode - $debug and print "${s}H $H", "\n"; + $debug and print "$s uid $m_uid header [$H]", "\n"; if ($skipheader and $H =~ m/$skipheader/i) { - $debug and print "Skipping header $H\n"; + $debug and print "$s uid $m_uid skipping header [$H]\n"; next; } $headstr .= "$H"; } } - if ( ( ! $headstr) and ( $takebody ) ){ - print "no header so taking body first 2Ko\n"; - $imap->fetch($m_uid, "BODY.PEEK[TEXT]<0.2048>"); - $headstr = $imap->_transaction_literals; - - if ( 4048 <= length( $headstr ) ) { - # the imap server might reply the whole message - # this is bad for memory on huge mailboxes - $takebody = 0 ; - $headstr = '' ; - $h1_msgs_copy_by_uid{ $m_uid } = 1 ; - } - } return() if ( ! $headstr ); my $size = $s_fir->{$m_uid}->{"RFC822.SIZE"}; @@ -3294,7 +3425,7 @@ sub parse_header_msg { my $idate = $s_fir->{$m_uid}->{"INTERNALDATE"}; $size = length($headstr) unless ($size); my $m_md5 = md5_base64($headstr); - $debug and print "$s msg $m_uid:$m_md5:$size\n"; + $debug and print "$s uid $m_uid sig $m_md5 size $size\n"; my $key; if ($skipsize) { $key = "$m_md5"; @@ -3375,7 +3506,7 @@ sub check_last_release { } sub imapsync_version { - my $rcs = '$Id: imapsync,v 1.422 2011/05/08 17:21:38 gilles Exp gilles $ '; + my $rcs = '$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ '; $rcs =~ m/,v (\d+\.\d+)/; my $VERSION = ($1) ? $1: "UNKNOWN"; return($VERSION); @@ -3678,7 +3809,7 @@ sub good_date { #print "internal: [$1][$2][$3][$4]\n"; my ($day_1, $date_rest, $hour, $zone) = ($1,$2,$3,$4); $day_1 = '0' if ($day_1 eq ''); - $zone = '' if not defined($zone); + $zone = ' +0000' if not defined($zone); $d = $day_1 . $date_rest . $hour . $zone; @@ -3686,7 +3817,7 @@ sub good_date { #print "header: [$1][$2][$3][$4][$5][$6]\n"; my ($day_1, $day_rest, $month, $year, $hour, $zone) = ($1,$2,$3,$4,$5,$6); $day_1 = '0' if ($day_1 eq ''); - $zone = '' if not defined($zone); + $zone = ' +0000' if not defined($zone); $d = $day_1 . "$day_rest-$month-$year" . $hour . $zone; }else{ @@ -3811,10 +3942,10 @@ sub tests_good_date { ok('' eq good_date(), 'good_date no arg'); ok('"24-Aug-2010 16:00:00 +0200"' eq good_date('24-Aug-2010 16:00:00 +0200'), 'good_date internal 2digit zone'); - ok('"24-Aug-2010 16:00:00"' eq good_date('24-Aug-2010 16:00:00'), 'good_date internal 2digit no zone'); + ok('"24-Aug-2010 16:00:00 +0000"' eq good_date('24-Aug-2010 16:00:00'), 'good_date internal 2digit no zone'); ok('"01-Sep-2010 16:00:00 +0200"' eq good_date( '1-Sep-2010 16:00:00 +0200'), 'good_date internal SP 1digit'); ok('"24-Aug-2010 16:00:00 +0200"' eq good_date('Tue, 24 Aug 2010 16:00:00 +0200'), 'good_date header 2digit zone'); - ok('"01-Sep-2010 16:00:00"' eq good_date('Wed, 1 Sep 2010 16:00:00'), 'good_date header SP 1digit zone'); + ok('"01-Sep-2010 16:00:00 +0000"' eq good_date('Wed, 1 Sep 2010 16:00:00'), 'good_date header SP 1digit zone'); ok('"01-Sep-2010 16:00:00 +0200"' eq good_date('Wed, 1 Sep 2010 16:00:00 +0200'), 'good_date header SP 1digit zone'); ok('"01-Sep-2010 16:00:00 +0200"' eq good_date('Wed, 1 Sep 2010 16:00:00 +0200 (CEST)'), 'good_date header SP 1digit zone'); @@ -3881,11 +4012,132 @@ sub delete_folders_in_2_not_in_1 { } } + +sub extract_header { + my $string = shift ; + + my ( $header ) = split( /\n\n/, $string ) ; + #print "[$header]\n" ; + return( $header ) ; +} + +sub tests_extract_header { + ok( +'Message-Id: <20100428101817.A66CB162474E@plume.est.belle> +Date: Wed, 28 Apr 2010 12:18:17 +0200 (CEST) +From: gilles@louloutte.dyndns.org (Gilles LAMIRAL)' +eq extract_header( +'Message-Id: <20100428101817.A66CB162474E@plume.est.belle> +Date: Wed, 28 Apr 2010 12:18:17 +0200 (CEST) +From: gilles@louloutte.dyndns.org (Gilles LAMIRAL) + +body +lalala +' ), 'extract_header: 1') ; + +} + +sub decompose_header{ + my $string = shift ; + + my $header = { } ; + + my ($key, $val ) ; + my @line = split( /\n|\r\n/, $string ) ; + foreach my $line ( @line ) { + #print "DDD $line\n" ; + if( $line =~ m/(^[^ :]+): (.*)/ ) { + $key = $1 ; + $val = $2 ; + #print "DDD [$key] [$val]\n" ; + push( @{ $header->{ $key } }, $val ) ; + }elsif( $line =~ m/^(\s+)(.*)/ ) { + $val = $2 ; + #print "DDD only [$val]\n" ; + @{ $header->{ $key } }[ -1 ] .= " $val" ; + }else{ + next ; + } + } + #require Data::Dumper ; + #print Data::Dumper->Dump( [ $header ] ) ; + + return( $header ) ; +} + + +sub tests_decompose_header{ + + my $header_dec ; + + $header_dec = decompose_header( +'KEY_1: VAL_1 +KEY_2: VAL_2 + VAL_2_+ + VAL_2_++ +KEY_3: VAL_3 +KEY_1: VAL_1_other +' + ) ; + + ok( 'VAL_3' + eq $header_dec->{ 'KEY_3' }[0], 'decompose_header: VAL_3' ) ; + + ok( 'VAL_1' + eq $header_dec->{ 'KEY_1' }[0], 'decompose_header: VAL_1' ) ; + + ok( 'VAL_1_other' + eq $header_dec->{ 'KEY_1' }[1], 'decompose_header: VAL_1_other' ) ; + + ok( 'VAL_2 VAL_2_+ VAL_2_++' + eq $header_dec->{ 'KEY_2' }[0], 'decompose_header: VAL_2 VAL_2_+ VAL_2_++' ) ; + + + $header_dec = decompose_header( +'Message-Id: <20100428101817.A66CB162474E@plume.est.belle> +Date: Wed, 28 Apr 2010 12:18:17 +0200 (CEST) +From: gilles@louloutte.dyndns.org (Gilles LAMIRAL)' + ) ; + + ok( '<20100428101817.A66CB162474E@plume.est.belle>' + eq $header_dec->{ 'Message-Id' }[0], 'decompose_header: 1' ) ; + + $header_dec = decompose_header( +'Return-Path: +Received: by plume.est.belle (Postfix, from userid 1000) + id 120A71624742; Wed, 28 Apr 2010 01:46:40 +0200 (CEST) +Subject: test:eekahceishukohpe' + ) ; + ok( +'by plume.est.belle (Postfix, from userid 1000) id 120A71624742; Wed, 28 Apr 2010 01:46:40 +0200 (CEST)' + eq $header_dec->{ 'Received' }[0], 'decompose_header: 2' ) ; + + $header_dec = decompose_header( +'Received: from plume (localhost [127.0.0.1]) + by plume.est.belle (Postfix) with ESMTP id C6EB73F6C9 + for ; Mon, 26 Nov 2007 10:39:06 +0100 (CET) +Received: from plume [192.168.68.7] + by plume with POP3 (fetchmail-6.3.6) + for (single-drop); Mon, 26 Nov 2007 10:39:06 +0100 (CET)' + ) ; + ok( + 'from plume (localhost [127.0.0.1]) by plume.est.belle (Postfix) with ESMTP id C6EB73F6C9 for ; Mon, 26 Nov 2007 10:39:06 +0100 (CET)' + eq $header_dec->{ 'Received' }[0], 'decompose_header: 3' ) ; + ok( + 'from plume [192.168.68.7] by plume with POP3 (fetchmail-6.3.6) for (single-drop); Mon, 26 Nov 2007 10:39:06 +0100 (CET)' + eq $header_dec->{ 'Received' }[1], 'decompose_header: 3' ) ; +} + + sub tests_debug { SKIP: { skip "No test in normal run" if ( not $tests_debug ); - tests_flagsCase( ) ; + tests_match_a_cache_file( ) ; + tests_cache_map( ) ; + tests_get_cache( ) ; + tests_clean_cache( ) ; + tests_clean_cache_2( ) ; } } @@ -3909,13 +4161,17 @@ sub tests { tests_imapsync_basename(); tests_list_keys_in_2_not_in_1(); tests_convert_sep_to_slash( ) ; + tests_match_a_cache_file( ) ; tests_cache_map( ) ; tests_get_cache( ) ; tests_clean_cache( ) ; - tests_match_a_cache_file( ) ; + tests_clean_cache_2( ) ; tests_touch( ) ; tests_ucsecond( ) ; - + tests_flagsCase( ) ; + tests_mkpath( ) ; + tests_extract_header( ) ; + tests_decompose_header( ) ; } } @@ -4121,7 +4377,7 @@ use constant NonFolderArg => 1; # Value to pass to Massage to my $headers = {}; # hash from message ids to header hash my $split = $self->Split() || scalar(@$msgspec_all); while(my @msgs = splice(@$msgspec_all, 0, $split)) { - $debug and print "SPLIT: @msgs\n"; + #$debug and print "SPLIT: @msgs\n"; my $msgspec = \@msgs; # Make $msg a comma separated list, of messages we want @@ -4384,7 +4640,18 @@ no warnings 'once'; return $self->{SSL}; }; +*Mail::IMAPClient::Starttls = sub { + my $self = shift; + + if (@_) { $self->{Starttls} = shift } + return $self->{Starttls}; +}; + + + *Mail::IMAPClient::exists = sub { + # Bad implementation STATUS fails and can close the connexion + # Exchange does this after 10 failures my ( $self, $folder ) = @_; $self->status($folder) ? $self : undef; }; diff --git a/index.shtml b/index.shtml index 6f3ada6..0c96237 100644 --- a/index.shtml +++ b/index.shtml @@ -5,7 +5,7 @@ Imapsync: an IMAP migration tool ( release <!--#exec cmd="cat VERSION"--> ) - + @@ -71,6 +71,20 @@ where the user plays independently on both sides. Use offlineimap

New features or bugfixes since previous release 1.411:

    +
  • 1.434
    • +
  • Bugfix: Changed the way imapsync knows whether a folder exists or not. Exchange might be happy and stop deconnecting for this reason.
    • +
  • Bugfix: Reconnections are well done in TLS mode now.
    • +
  • Bugfix: IMAP RFC 3501 and some imap servers require internal dates have a zone data. Default to +0000 when host1 doesn't give it.
    • +
  • Bugfix: Options --maxsize --minsize now really work with --useuid (1.422 had a cache issue).
    • +
  • Bugfix: Improved the way imapsync deals with headers:
      +
    • Stopped getting first 2KB body of message. Not a good idea in practice.
      • +
    • If massive get headers fails then take the whole header one by one (instead of body).
      • +
    • Default is like --useheader "Message-Id" --useheader "Message-ID" (instead of just "Message-Id").
      • +
    • Use header "Message-Id" and header "Date" as md5 signature when taking the whole header.
      • +
    +
    • + +
  • 1.422
  • Better default behavior: Option --delete2 implies --expunge2 now (unless --noexpunge2 is given.)
  • Better default behavior: Correct flags case to be RFC compliant on host2 if host1 is not (\SEEN -> \Seen)
  • Better debug: Added --debugcontent option to avoid debugging content (can be big) with --debug option.
    • @@ -79,7 +93,7 @@ where the user plays independently on both sides. Use offlineimap
  • Bugfix: Options --maxsize --minsize now work with --useuid
  • Bugfix: Flag sync of already transfered messages now take care of --maxsize --minsize options
  • Bugfix: Added 0 length message tracking when fetching host1 (to avoid frequently "APPEND {0}" recent issues).
    • -
  • Bugfix: Avoid now Inbox <-> INBOX problem ("already exists").
    • +
  • Bugfix: Avoid Inbox vs INBOX case problem ("already exists").
  • Bugfix: --proxyauth2 was setting proxyauth1 instead of proxyauth2
@@ -356,23 +370,25 @@ will be to code it or fix it.
  • Deerfield VisNetic MailServer 5.8.6 [host1]
  • dkimap4 [host1]
  • Domino (Notes) 4.61[host1], 6.5, 5.0.6, 5.0.7, 7.0.2, 6.0.2CF1, 7.0.1[host1], 8.0.1[host1]
    • -
  • Dovecot 0.99.10.4, 0.99.14, 0.99.14-8.fc4, 1.0-0.beta2.7, +
  • Dovecot 0.99.10.4, 0.99.14, 0.99.14-8.fc4, 1.0-0.beta2.7, 1.0.0 [dest/source] (LGPL) (http://www.dovecot.org/)
  • Eudora WorldMail v2
    • -
  • Gimap (Gmail imap) [host1] [host2]
    • +
  • Gimap (Gmail imap) [host1] [host2]
  • GMX IMAP4 StreamProxy.
  • Godaddy IMAP (since Godaddy runs Courier)
  • Groupwise IMAP (Novell) 6.x and 7.0. Buggy so see the FAQ.
  • hMailServer 5.3.3 [host2], 4.4.1 [host1], HMAILSERVER 5.3.2-B1769 on windows 2003 [hsot2]
  • iPlanet Messaging server 4.15, 5.1, 5.2
  • IMail 7.15 (Ipswitch/Win2003), 8.12, 11.03 [host1]
    • -
  • MailEnable 4.23 [host1] [host2]
    • +
  • MailEnable 4.23 [host1][host2], 4.26 [host1][host2]
  • MDaemon 7.0.1, 8.0.2, 8.1, 9.5.4 (Windows server 2003 R2 platform), 12 [host2]
  • Mercury 4.1 (Windows server 2000 platform)
    • -
  • Microsoft Exchange Server 5.5, 6.0.6249.0[host1], 6.0.6487.0[host1], +
  • Microsoft Exchange Server 5.5, 6.0.6249.0[host1], 6.0.6487.0[host1], 6.5.7638.1 [host2], 6.5 [host1], Exchange 2007 SP1 (with Update Rollup 2), Exchange2007-EP-SP2, - Exchange 2010 RTM (Release to Manufacturing) [host2]
    • + Exchange 2010 RTM (Release to Manufacturing) [host2], + Exchange 2010 SP1 RU2 [host2] +
  • Mirapoint server
  • Netscape Mail Server 3.6 (Wintel)
  • Netscape Messaging Server 4.15 Patch 7
    • @@ -392,7 +408,8 @@ will be to code it or fix it.
    (http://www.washington.edu/imap/)
  • UW - QMail v2.1
  • VMS, Imap part of TCP/IP suite of VMS 7.3.2
    • -
  • Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5, 6.x
    • +
  • Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, + Zimbra 5.0.24_GA_3356.RHEL4 [host1], 5.5, 6.x
    • @@ -421,7 +438,7 @@ alt="Viewable With Any Browser" /> This document last modified on -($Id: index.shtml,v 1.66 2011/05/09 00:45:40 gilles Exp gilles $) +($Id: index.shtml,v 1.69 2011/05/16 17:10:13 gilles Exp gilles $)

    diff --git a/paypal_return.shtml b/paypal_return.shtml index 3bae177..2acf084 100644 --- a/paypal_return.shtml +++ b/paypal_return.shtml @@ -5,7 +5,7 @@ imapsync download - + @@ -31,11 +31,7 @@ border:0px; - - - -

    imapsync download

    I thank you for buying Imapsync!

    @@ -46,12 +42,12 @@ You may log into your account at www.paypal.com to view details of this transaction.

    -

    You will find the latest imapsync source code release 1.417 at the following link:
    -    http://www.linux-france.org/depot/2011_05_07/8qkE2L/ +

    You will find the latest imapsync source code release 1.422 at the following link:
    +http://www.linux-france.org/depot/2011_05_09/EocZFt/

    -

    You will find the latest imapsync.exe binary release 1.417 at the following link:
    -http://www.linux-france.org/depot/2011_05_07/eQ5bXu/ +

    You will find the latest imapsync.exe binary release 1.422 at the following link:
    +http://www.linux-france.org/depot/2011_05_09/XhVbYj/

    You will receive an invoice soon.

    @@ -59,8 +55,13 @@ to view details of this transaction.

    Next imapsync releases will be available for one year without extra payment.
    I will send you a message explaining how to get them

    -

    I thank you again for buying and using imapsync, -I wish you successful imap transfers!

    +

    To avoid loosing time, explain your specific needs, find best solutions
    +and then succeed your migration you can buy professionnal support at the link
    +http://www.linux-france.org/prj/imapsync/#buy_support +

    + +

    I thank you again for buying and using imapsync,
    +I wish you successful imap transfers!

    imapsync homepage

    @@ -82,7 +83,7 @@ gilles.lamiral@laposte.net

    This document last modified on
    -($Id: paypal_return.shtml,v 1.5 2011/05/07 02:56:35 gilles Exp gilles $) +($Id: paypal_return.shtml,v 1.7 2011/05/16 17:23:12 gilles Exp gilles $)

    diff --git a/test2.bat b/test2.bat index e23f935..f21a1c0 100755 --- a/test2.bat +++ b/test2.bat @@ -1,5 +1,5 @@ -REM $Id: test.bat,v 1.8 2011/01/15 06:30:33 gilles Exp gilles $ +REM $Id: test2.bat,v 1.2 2011/05/16 16:41:47 gilles Exp gilles $ cd C:\msys\1.0\home\Admin\imapsync REM perl ./imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --delete2 --expunge2 --folder INBOX @@ -9,6 +9,13 @@ REM perl ./imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p - REM imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --justfolders --nofoldersize --folder INBOX.yop.yap --sep1 / --regextrans2 "s,/,_," -imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --nofoldersize --folder INBOX.yop.yap --regexflag 's/\\Answered//g' --debug > out.txt +REM imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --nofoldersize --folder INBOX.yop.yap --regexflag 's/\\Answered//g' --debug > out.txt +REM perl imapsync --version +REM perl imapsync --tests_debug + +imapsync.exe ^ + --host1 p --user1 big1 --passfile1 secret.big1 ^ + --host2 p --user2 big2 --passfile2 secret.big2 ^ + --folder INBOX.bigmail diff --git a/tests.sh b/tests.sh index 67b9528..557985e 100644 --- a/tests.sh +++ b/tests.sh @@ -1,6 +1,6 @@ #!/bin/sh -# $Id: tests.sh,v 1.163 2011/05/09 00:10:16 gilles Exp gilles $ +# $Id: tests.sh,v 1.165 2011/05/16 16:41:11 gilles Exp gilles $ # Example 1: # CMD_PERL='perl -I./Mail-IMAPClient-3.25/lib' sh -x tests.sh @@ -264,7 +264,7 @@ ll_folder_create_INBOX_Inbox() { --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ --folder INBOX --regextrans2 's/INBOX/Inbox/' \ - --justfolders + --justfolders --nofoldersizes } @@ -287,6 +287,7 @@ ll_debugimap() { --passfile2 ../../var/pass/secret.titi \ --folder INBOX.oneemail --debugimap } + ll_few_emails() { $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 tata \ @@ -296,6 +297,38 @@ ll_few_emails() { --folder INBOX.few_emails } +ll_noheader() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.few_emails --useheader '' +} + +ll_noheader_force() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.few_emails \ + --useheader '' \ + --skipheader 'Message-Id|Date' +} + + + +ll_usecachemaxage() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --useuid --maxage 3 +} + + ll_folderrec() { $CMD_PERL ./imapsync \ @@ -523,6 +556,19 @@ ll_justfoldersizes() --justfoldersizes } +ll_justfoldersizes_noexist() +{ + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --justfoldersizes --folder NoExist --folder INBOX +} + + + + ll_dev_reconnect() { # in another terminal: @@ -580,6 +626,35 @@ EOF --delete2 } +ll_dev_reconnect_tls() +{ +# in another terminal: +# +: <<'EOF' +while :; do + killall -v -u vmail imapd; + RAND_WAIT=`numrandom .1..5i.1` + echo sleeping $RAND_WAIT + sleepenh $RAND_WAIT +done +# or +while read y; do + echo ENTER to kill all imapd + killall -v -u vmail imapd; +done + +EOF + can_send && sendtestmessage +# can_send && sendtestmessage + $CMD_PERL ./imapsync \ + --host1 $HOST1 --tls1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --tls2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX --useuid \ + --delete2 --debugsleep --debugimap +} + @@ -646,7 +721,8 @@ ll_newmessage() --passfile1 ../../var/pass/secret.tata \ --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ - --maxage 1 --folder INBOX --nofoldersizes --noreleasecheck + --maxage 1 --folder INBOX --nofoldersizes --noreleasecheck \ + --debugLIST } ll_folder_INBOX() @@ -694,9 +770,23 @@ ll_maxsize_useuid() --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ --maxsize 10 --nofoldersizes --folder INBOX \ - --useuid + --useuid --debugcache } +ll_minsize_useuid() +{ + can_send && sendtestmessage + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --nofoldersizes --folder INBOX \ + --useuid --debugLIST --minsize 500 --maxage 1 +} + + + ll_skipsize() { @@ -2039,6 +2129,8 @@ ll_folder_create_INBOX_Inbox ll_delete2folders ll_useuid ll_useuid_nousecache +ll_noheader_force +ll_noheader ' other_tests=' diff --git a/tmp/cache/F1/F2/100_200 b/tmp/cache/F1/F2/100_200 deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/cache/F1/F2/101_201 b/tmp/cache/F1/F2/101_201 deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/cache/F1/F2/120_220 b/tmp/cache/F1/F2/120_220 deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/cache/F1/F2/155_255 b/tmp/cache/F1/F2/155_255 deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/cache/F1/F2/177_277 b/tmp/cache/F1/F2/177_277 deleted file mode 100644 index e69de29..0000000