1.434

ChangeLog

@@ -1,17 +1,74 @@
 
 RCS file: RCS/imapsync,v
 Working file: imapsync
-head: 1.422
+head: 1.434
 branch:
 locks: strict
-	gilles: 1.422
+	gilles: 1.434
 access list:
 symbolic names:
 keyword substitution: kv
-total revisions: 422;	selected revisions: 422
+total revisions: 434;	selected revisions: 434
 description:
 ----------------------------
-revision 1.422	locked by: gilles;
+revision 1.434	locked by: gilles;
+date: 2011/05/16 07:16:19;  author: gilles;  state: Exp;  lines: +142 -57
+Bugfix. Made --usecache work with --maxage or --maxsize or --min*
+----------------------------
+revision 1.433
+date: 2011/05/16 03:07:59;  author: gilles;  state: Exp;  lines: +191 -49
+Improved the way imapsync deals with headers:
+ - Stopped getting first 2KB of message. Not a good idea.
+ - If $imap2->parse_headers() fails then take the whole header (instead of body).
+ - Default is like --useheader Message-Id --useheader Message-ID
+ - Use header Message-Id and header Date as sig md5 when taking the whole header.
+
+Better output in debug mode.
+----------------------------
+revision 1.432
+date: 2011/05/15 22:06:19;  author: gilles;  state: Exp;  lines: +8 -7
+Options --usecache and --maxsize --minsize can safely be used if --delete is there
+----------------------------
+revision 1.431
+date: 2011/05/15 00:47:03;  author: gilles;  state: Exp;  lines: +17 -8
+Added tests of mkpath very long path > 300 char. Win32 fails on them.
+----------------------------
+revision 1.430
+date: 2011/05/14 23:13:06;  author: gilles;  state: Exp;  lines: +14 -9
+Bugfix. Added special case for Inbox vs INBOX bug creation ("Couldn't create folder [Inbox] from [INBOX]: 143 NO INBOX already exists!")
+----------------------------
+revision 1.429
+date: 2011/05/14 21:24:31;  author: gilles;  state: Exp;  lines: +8 -8
+Adapted regression tests for good_date() when no zone is given.
+----------------------------
+revision 1.428
+date: 2011/05/14 02:31:02;  author: gilles;  state: Exp;  lines: +8 -8
+Bugfix. intarnal date needs zone data. Default to  +0000.
+----------------------------
+revision 1.427
+date: 2011/05/14 01:48:07;  author: gilles;  state: Exp;  lines: +15 -13
+Bugfix. Starttls() only for 2.2.9
+----------------------------
+revision 1.426
+date: 2011/05/14 01:14:16;  author: gilles;  state: Exp;  lines: +8 -8
+Fix. Removed a debug print always printed.
+----------------------------
+revision 1.425
+date: 2011/05/14 00:33:46;  author: gilles;  state: Exp;  lines: +16 -15
+Bugfix. Changed the way imapsync knows whether a folder exists or not. Exchange might be happy and stop deconnecting for this reason.
+----------------------------
+revision 1.424
+date: 2011/05/14 00:12:38;  author: gilles;  state: Exp;  lines: +20 -9
+Added a warning and die if --usecache and one of  --maxsize--minsize --maxage --minage is used.
+----------------------------
+revision 1.423
+date: 2011/05/13 22:26:01;  author: gilles;  state: Exp;  lines: +23 -12
+Bugfix. Reconnections are well done in tls mode now.
+Zimbra 5.0.24_GA_3356.RHEL4 [host1]
+Exchange 2010 SP1 RU2 [host2]
+Added --debugsleep to have to play will kill and reconnections.
+----------------------------
+revision 1.422
 date: 2011/05/08 17:21:38;  author: gilles;  state: Exp;  lines: +17 -12
 Added --debugLIST to track messages list uid or number only.
 Bugfix: a lack of variable initialisation caused to fetch no existing messages.

FAQ

@@ -1,5 +1,5 @@
 #!/bin/cat
-# $Id: FAQ,v 1.85 2011/02/28 16:02:17 gilles Exp gilles $
+# $Id: FAQ,v 1.86 2011/05/16 16:43:12 gilles Exp gilles $
 
 +------------------+
 | FAQ for imapsync |
@@ -533,14 +533,21 @@ R. Do not use the --folder option.
 Q. How to migrate from or to Exchange 2007/2010 with an 
    admin/authuser account?
    
-R. The trick comes from Michele Marcionelli:
+R. The tricks comes from Michele Marcionelli and Benjamin Priestman:
 
 This doesn't work:
-imapsync ... --user2 user2 --authuser2 admin2 --password2 adminpassword2 ...
+  imapsync ... --user2 user2 --authuser2 admin2 --password2 adminpassword2 ...
 
-This works:
+This might works:
-imapsync ... --user2 'domain\admin2\user2' --password2 adminpassword2 ...
+  imapsync ... --user2 'domain\admin2\user2' --password2 adminpassword2 ...
+or 
+  imapsync ... --user2 'admin2@domain\user2' --password2 adminpassword2 ...
 
+where "domain" is set be the user's UPN in Active Directory
+or the NETBIOS or DNS name of the domain.
+
+The exact format might vary depending on local configuration and you
+should experiment with the different formats.
 
 ======================================================================
 Q. How to migrate from uw-imap with an admin/authuser account?
@@ -845,6 +852,15 @@ format issues. And now it works fine. (Thanks to Hansjoerg.Maurer)
               Server specific issues and solutions
 =======================================================================
 
+=======================================================================
+Q. From Zimbra to XXX
+
+imapsync ... \
+	--exclude "Conversation Action Settings" \
+	--exclude "Quick Step Settings" \
+	--exclude "News Feed"
+
+=======================================================================
 Q. From or to HMailServer version 4.4.1.
 
 R. You have to add prefix and separator manually because  4.4.1 doesn't

Makefile

@@ -1,5 +1,5 @@
 
-# $Id: Makefile,v 1.72 2011/05/09 00:11:00 gilles Exp gilles $	
+# $Id: Makefile,v 1.74 2011/05/16 17:25:22 gilles Exp gilles $	
 
 .PHONY: help usage all
 
@@ -10,6 +10,7 @@ usage:
 	@echo "make install # as root"
 	@echo "make testf   # run tests"
 	@echo "make testv   # run tests verbosely"
+	@echo "make test_quick # few tests verbosely"
 	@echo "make test3xx # run tests with (last) Mail-IMAPClient-3.xy"
 	@echo "make test229 # run tests with Mail-IMAPClient-2.2.9"
 	@echo "make tests_win32 # run tests on win32"
@@ -84,13 +85,13 @@ cidone:
 
 .PHONY: test tests testp testf test3xx
 
-test_quick : test_quick_229 test_quick_3xx
+test_quick : test_quick_3xx test_quick_229 
 
 test_quick_229: imapsync tests.sh
-	CMD_PERL='perl -I./Mail-IMAPClient-2.2.9' /usr/bin/time sh tests.sh locallocal 1>/dev/null
+	CMD_PERL='perl -I./Mail-IMAPClient-2.2.9' /usr/bin/time sh -x tests.sh locallocal
 
 test_quick_3xx: imapsync tests.sh
-	CMD_PERL='perl -I./Mail-IMAPClient-3.28/lib' /usr/bin/time sh tests.sh locallocal 1>/dev/null
+	CMD_PERL='perl -I./Mail-IMAPClient-3.28/lib' /usr/bin/time sh -x tests.sh locallocal
 
 testv:
 	sh -x tests.sh
@@ -192,7 +193,7 @@ lfo: cidone  niouze_lfo upload_lfo
 
 dist: cidone test clean all INSTALL tarball
 
-tarball: cidone all imapsync.exe
+tarball: cidone all
 	echo making tarball $(DIST_FILE)
 	mkdir -p dist
 	mkdir -p ../prepa_dist/$(DIST_NAME)

README

@@ -3,7 +3,7 @@ NAME
     Synchronise mailboxes between two imap servers. Good at IMAP migration.
     More than 36 different IMAP server softwares supported with success.
 
-    $Revision: 1.422 $
+    $Revision: 1.434 $
 
 SYNOPSIS
     To synchronise imap account "foo" on "imap.truc.org" to imap account
@@ -336,8 +336,9 @@ IMAP SERVERS
      - Mercury 4.1 (Windows server 2000 platform)
      - Microsoft Exchange Server 5.5, 6.0.6249.0[host1], 6.0.6487.0[host1], 
        6.5.7638.1 [host2], 6.5 [host1], Exchange 2007 SP1 (with Update Rollup 2), 
-       Exchange2007-EP-SP2, 
+       Exchange2007-EP-SP2,
-       Exchange 2010 RTM (Release to Manufacturing) [host2]
+       Exchange 2010 RTM (Release to Manufacturing) [host2],
+       Exchange 2010 SP1 RU2[host2],
      - Mirapoint
      - Netscape Mail Server 3.6 (Wintel !)
      - Netscape Messaging Server 4.15 Patch 7
@@ -357,7 +358,8 @@ IMAP SERVERS
        (http://www.washington.edu/imap/)
      - UW - QMail v2.1
      - Imap part of TCP/IP suite of VMS 7.3.2
-     - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5, 6.x
+     - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 
+       Zimbra 5.0.24_GA_3356.RHEL4 [host1], 5.5, 6.x
 
     Please report to the author any success or bad story with imapsync and
     do not forget to mention the IMAP server software names and version on
@@ -422,5 +424,5 @@ SIMILAR SOFTWARES
 
     Feedback (good or bad) will often be welcome.
 
-    $Id: imapsync,v 1.422 2011/05/08 17:21:38 gilles Exp gilles $
+    $Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $
 

TIME

@@ -1,3 +1,9 @@
+540 (1.434) (1.433) (1.432) 
+180 Tests of mkpath very long path > 300 char. Win32 fails on them. (1.431)
+    Added special case for Inbox vs INBOX bug creation. (1.430)
+    Adapted regression tests for good_date() when no zone is given. (1.429)
+ 60 Reading, replying and fixing from Benjamin Priestman feedback.
+ 90 Various email help.
 300 Release 1.417. Some numbers section. INSTALL file.
  60 Bugfix. --maxsize --minsize now work with --useuid + flag sync of already transfered messages now take care of --maxsize --minsize options.
 120 Exit on --delete --delete2. --expunge1 same as --expunge. --delete2 implies --expunge2.

TODO

@@ -1,5 +1,5 @@
 #!/bin/cat
-# $Id: TODO,v 1.96 2011/04/26 10:48:03 gilles Exp gilles $
+# $Id: TODO,v 1.97 2011/05/16 16:39:38 gilles Exp gilles $
 
 TODO file for imapsync
 ----------------------
@@ -23,10 +23,8 @@ Evaluate
 http://www.rackspace.com/apps/email_hosting/migrations
 http://www.yippiemove.com/
 
-Fix Exchange 2010 SP1 issue with --foldersizes when
+Fix long path over than 256 character on Win32. 
-host2 folders don't exist. $imap->exists calls STATUS.
+Think about Digest::SHA or Digest::SHA::PurePerl.
-Is it RFC compliant or an Exchange bug?
-Exchange quit after 10 errors.
 
 Fix "\Forwarded" flag bug in courier.
 Does \lalala can be forbidden (courier does a 
@@ -35,7 +33,6 @@ with
 * OK [PERMANENTFLAGS (\* \Draft \Answered \Flagged \Deleted \Seen)] Limited
 
 
-
 Suggestion: it's very difficult to track down messages which are behaving
 funny during the sync. It would be great - and presumably easy to code -
 to have an option to have imapsync display e.g. the subject of an
@@ -159,6 +156,13 @@ http://asg.web.cmu.edu/cyrus/download/imapd/altnamespace.html
 
 ===========================================================================
 
+DONE. Make --usecache works with --maxage --maxsize etc.
+
+DONE. Fix Exchange 2010 SP1 issue with --foldersizes when
+host2 folders don't exist. $imap->exists calls STATUS.
+Is it RFC compliant or an Exchange bug?
+Exchange quit after 10 errors.
+
 DONE. Make --delete2 works with --useuid
 
 DONE. write a comment to http://blog.migrationwiz.com/2010/12/09/imapsync-vs-migrationwiz/

VERSION

@@ -1 +1 @@
-1.422
+1.434

VERSION_EXE

@@ -1 +1 @@
-1.422
+1.434

imapsync

@@ -20,7 +20,7 @@ Synchronise mailboxes between two imap servers.
 Good at IMAP migration. More than 36 different IMAP server softwares
 supported with success.
 
-$Revision: 1.422 $
+$Revision: 1.434 $
 
 =head1 SYNOPSIS
 
@@ -384,8 +384,9 @@ Success stories reported with the following 41 imap servers
  - Mercury 4.1 (Windows server 2000 platform)
  - Microsoft Exchange Server 5.5, 6.0.6249.0[host1], 6.0.6487.0[host1], 
    6.5.7638.1 [host2], 6.5 [host1], Exchange 2007 SP1 (with Update Rollup 2), 
-   Exchange2007-EP-SP2, 
+   Exchange2007-EP-SP2,
-   Exchange 2010 RTM (Release to Manufacturing) [host2]
+   Exchange 2010 RTM (Release to Manufacturing) [host2],
+   Exchange 2010 SP1 RU2[host2],
  - Mirapoint
  - Netscape Mail Server 3.6 (Wintel !)
  - Netscape Messaging Server 4.15 Patch 7
@@ -405,7 +406,8 @@ Success stories reported with the following 41 imap servers
    (http://www.washington.edu/imap/)
  - UW - QMail v2.1
  - Imap part of TCP/IP suite of VMS 7.3.2
- - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5, 6.x
+ - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 
+   Zimbra 5.0.24_GA_3356.RHEL4 [host1], 5.5, 6.x
 
 Please report to the author any success or bad story with
 imapsync and do not forget to mention the IMAP server
@@ -496,7 +498,7 @@ Entries for imapsync:
 
 Feedback (good or bad) will often be welcome.
 
-$Id: imapsync,v 1.422 2011/05/08 17:21:38 gilles Exp gilles $
+$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $
 
 =cut
 
@@ -541,7 +543,7 @@ use constant {
 my(
         $rcs, $pidfile,
 	$debug, $debugimap, $debugimap1, $debugimap2, $debugcontent, $debugflags,
-	$debugLIST,
+	$debugLIST, $debugsleep, 
 	$nb_errors,
 	$host1, $host2, $port1, $port2,
 	$user1, $user2, $domain1, $domain2, 
@@ -601,14 +603,14 @@ my(
 	$modules_version,
 	$delete2folders, $delete2foldersonly, $delete2foldersbutnot,
 	$usecache, $debugcache, 
-	$takebody, %h1_msgs_copy_by_uid, $useuid, $h2_uidguess,
+	$wholeheaderifneeded, %h1_msgs_copy_by_uid, $useuid, $h2_uidguess,
 );
 
 # main program
 
 # global variables initialisation
 
-$rcs = '$Id: imapsync,v 1.422 2011/05/08 17:21:38 gilles Exp gilles $ ';
+$rcs = '$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ ';
 
 $total_bytes_transferred   = 0;
 $total_bytes_skipped = 0;
@@ -650,7 +652,7 @@ $pidfile ||= $tmpdir . '/imapsync.pid';
 # allow Mail::IMAPClient 3.0.xx by default
 $allow3xx = defined($allow3xx) ? $allow3xx : 1;
 
-$takebody  = defined( $takebody )  ? $takebody  : 1;
+$wholeheaderifneeded  = defined( $wholeheaderifneeded )  ? $wholeheaderifneeded  : 1;
 
 # turn on RFC standard flags correction like \SEEN -> \Seen
 $flagsCase = defined( $flagsCase ) ? $flagsCase : 1 ;
@@ -740,14 +742,12 @@ if ( $delete and $delete2 ) {
 	exit_clean( 0 ) ;
 }
 
-
+if ($idatefromheader) {
-
-if($idatefromheader) {
 	print "Turned ON idatefromheader, ",
 	      "will set the internal dates on host2 from the 'Date:' header line.\n";
 	$syncinternaldates = 0;
-
 }
+
 if ($syncinternaldates) {
 	print "Turned ON syncinternaldates, ",
 	      "will set the internal dates (arrival dates) on host2 same as host1.\n";
@@ -755,19 +755,19 @@ if ($syncinternaldates) {
 	print "Turned OFF syncinternaldates\n";
 }
 
-if(defined($authmd5) and ($authmd5)) {
+if (defined($authmd5) and ($authmd5)) {
 	$authmd51 = 1 ;
 	$authmd52 = 1 ;
 }
 
-if(defined($authmd51) and ($authmd51)) {
+if (defined($authmd51) and ($authmd51)) {
 	$authmech1 ||= 'CRAM-MD5';
 }
 else{
 	$authmech1 ||= $authuser1 ? 'PLAIN' : 'LOGIN';
 }
 
-if(defined($authmd52) and ($authmd52)) {
+if (defined($authmd52) and ($authmd52)) {
 	$authmech2 ||= 'CRAM-MD5';
 }
 else{
@@ -800,7 +800,19 @@ $fastio2 = (defined($fastio2)) ? $fastio2 : 0;
 $reconnectretry1 = (defined($reconnectretry1)) ? $reconnectretry1 : 3;
 $reconnectretry2 = (defined($reconnectretry2)) ? $reconnectretry2 : 3;
 
-@useheader = ("Message-Id") unless (@useheader);
+@useheader = ( "Message-Id", "Message-ID" ) unless ( @useheader ) ;
+
+my %useheader ;
+
+# Make a hash %useheader of each --useheader 'key' in uppercase
+@useheader{ map( { uc( $_ ) } @useheader ) } = (  ) ;
+
+my %useheaderclassic ;
+@useheaderclassic{ qw(MESSAGE-ID DATE) } = (  ) ;
+
+#require Data::Dumper ;
+#print Data::Dumper->Dump( [ \%useheader ] ) ;
+
 
 print "Host1: imap server [$host1] port [$port1] user [$user1]\n";
 print "Host2: imap server [$host2] port [$port2] user [$user2]\n";
@@ -817,6 +829,18 @@ $password2 || $passfile2 || do {
 
 $password2 = (defined($passfile2)) ? firstline ($passfile2) : $password2;
 
+if ( ! ( 1
+      or ( $maxsize 
+      or $minsize 
+      or $maxage  
+      or $minage )
+     and $usecache 
+     and ! $delete ) ) {
+        die_clean( 
+"Problem --usecache can not be used safely with options --maxsize--minsize --maxage --minage
+Use --nousecache or suppress the --max* --min* options\n" ) ;    
+}
+
 my $imap1 = ();
 my $imap2 = ();
 
@@ -1032,15 +1056,21 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 	
 	next FOLDER if ($justfolders);
 
-	my @h1_msgs = select_msgs($imap1);
+        my $h1_msgs_all_hash_ref = {  } ;
+	my @h1_msgs = select_msgs( $imap1, $h1_msgs_all_hash_ref );
 
-	( $debug or $debugLIST )  and print "LIST Host1: ", scalar(@h1_msgs), " messages [@h1_msgs]\n";
+	( $debug or $debugLIST ) and print "Host1 LIST: ", scalar( @h1_msgs ), " messages [@h1_msgs]\n" ;
+	#( $debug or $debugLIST ) and print "Host1 LIST: ", scalar( @$h1_msgs_all_ref ), " messages [@$h1_msgs_all_ref]\n" ;
 	# internal dates on host2 are after the ones on host1
 	# normally...
-	my @h2_msgs = select_msgs($imap2);
+        
+        my $h2_msgs_all_hash_ref = {  } ;
+	my @h2_msgs = select_msgs( $imap2, $h2_msgs_all_hash_ref ) ;
 	
-	( $debug or $debugLIST ) and print "LIST Host2: ", scalar(@h2_msgs), " messages [@h2_msgs]\n";
+	( $debug or $debugLIST ) and print "Host2 LIST: ", scalar(@h2_msgs), " messages [@h2_msgs]\n";
-
+        #( $debug or $debugLIST ) and print "Host2 LIST ALL: ", scalar( keys %$h2_msgs_all_hash_ref ), " messages [", 
+        #  join( ' ', keys( %$h2_msgs_all_hash_ref ) ), "]\n" ;
+        
 	my $cache_base = "$tmpdir/imapsync_cache/$host1/$user1/$host2/$user2";
 	my $cache_dir = cache_folder( $cache_base, $h1_fold, $h2_fold );
 	my ( $cache_1_2_ref, $cache_2_1_ref ) = ( {}, {} );
@@ -1048,7 +1078,8 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 	if ( $usecache ) {
 		print "cache directory: $cache_dir\n" ;
 		mkpath( "$cache_dir" ) ;
-		( $cache_1_2_ref, $cache_2_1_ref ) = get_cache($cache_dir, \@h1_msgs, \@h2_msgs) if ($usecache) ;
+		( $cache_1_2_ref, $cache_2_1_ref ) 
+                = get_cache( $cache_dir, \@h1_msgs, \@h2_msgs, $h1_msgs_all_hash_ref, $h2_msgs_all_hash_ref ) ;
 		print "CACHE h1 h2: ", scalar( keys %$cache_1_2_ref ), " files\n" ; 
 		$debug and print '[',
 		    map ( { "$_->$cache_1_2_ref->{$_} " } keys %$cache_1_2_ref ), " ]\n";
@@ -1058,16 +1089,16 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 	my %h1_hash = ();
 	my %h2_hash = ();
 	
-	my ( %h1_msgs_all, %h2_msgs_all ) ;
+	my ( %h1_msgs, %h2_msgs ) ;
-	@h1_msgs_all{ @h1_msgs } = ();
+	@h1_msgs{ @h1_msgs } = ();
-	@h2_msgs_all{ @h2_msgs } = ();
+	@h2_msgs{ @h2_msgs } = ();
 	
 	my @h1_msgs_in_cache = sort { $a <=> $b } keys %$cache_1_2_ref ;
 	my @h2_msgs_in_cache = keys %$cache_2_1_ref ;
 	
 	my ( %h1_msgs_no_cache, %h2_msgs_no_cache ) ;
-	%h1_msgs_no_cache = %h1_msgs_all ;
+	%h1_msgs_no_cache = %h1_msgs ;
-	%h2_msgs_no_cache = %h2_msgs_all ;
+	%h2_msgs_no_cache = %h2_msgs ;
 	delete @h1_msgs_no_cache{ @h1_msgs_in_cache } ;
 	delete @h2_msgs_no_cache{ @h2_msgs_in_cache } ;
 	
@@ -1088,16 +1119,18 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 		#print "delete2: @h2_msgs_delete2_no_cache\n";
 	}
 	
-	$debug and print "Host1 folder [$h1_fold] parsing headers\n";
+	$debug and print "Host1 parsing headers of folder [$h1_fold]\n";
 
 	my ($h1_heads_ref, $h1_fir_ref) = ({}, {});
 	$h1_heads_ref = $imap1->parse_headers([@h1_msgs_no_cache], @useheader) if (@h1_msgs_no_cache);
-	$debug and print "Time headers: ", timenext(), " s\n";
+	$debug and print "Host1 parsing headers of folder [$h1_fold] took ", timenext(), " s\n";
 
 	@$h1_fir_ref{@h1_msgs} = (undef);
+        
+	$debug and print "Host1 getting flags idate and sizes of folder [$h1_fold]\n" ;
 	$h1_fir_ref = $imap1->fetch_hash_2("FLAGS", "INTERNALDATE", "RFC822.SIZE", $h1_fir_ref)
 	  if (@h1_msgs);
-	$debug and print "Time fir: ", timenext(), " s\n";
+	$debug and print "Host1 getting flags idate and sizes of folder [$h1_fold] took ", timenext(), " s\n";
 	unless ($h1_fir_ref) {
 		warn
 		"Host1 folder $h1_fold: Could not fetch_hash_2 ",
@@ -1108,10 +1141,10 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 
 	my @h1_msgs_duplicate;
 	foreach my $m (@h1_msgs_no_cache) {
-		my $rc = parse_header_msg($imap1, $m, $h1_heads_ref, $h1_fir_ref, "F", \%h1_hash);
+		my $rc = parse_header_msg($imap1, $m, $h1_heads_ref, $h1_fir_ref, 'Host1', \%h1_hash);
 		if (! defined($rc)) {
 			my $h1_size = $h1_fir_ref->{$m}->{"RFC822.SIZE"} || 0;
-			print "+ Skipping msg #$m:$h1_size on host1 folder $h1_fold (no header so we ignore this message)\n";
+			print "Host1 $h1_fold/$m size $h1_size ignored (no header so we ignore this message)\n";
 			$total_bytes_skipped += $h1_size;
 			$nb_msg_skipped += 1;
 			$h1_nb_msg_noheader +=1;
@@ -1125,25 +1158,27 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 			$h1_nb_msg_duplicate += 1;
 		}
 	}
-	$debug and print "Time parsing headers on host1: ", timenext(), " s\n";
+	$debug and print "Host1 whole time parsing headers took ", timenext(), " s\n";
-	
+	$debug and print "\n";
-	$debug and print "Host2 folder [$h2_fold] parsing headers\n";
+        
-
+	$debug and print "Host2 parsing headers of folder [$h2_fold]\n";
+        
 	my ($h2_heads_ref, $h2_fir_ref) = ( {}, {} );
 	$h2_heads_ref =   $imap2->parse_headers([@h2_msgs_no_cache], @useheader) if (@h2_msgs_no_cache);
-	$debug and print "Time headers: ", timenext(), " s\n";
+	$debug and print "Host2 parsing headers of folder [$h2_fold] took ", timenext(), " s\n" ;
 
+	$debug and print "Host2 getting flags idate and sizes of folder [$h2_fold]\n" ;
 	@$h2_fir_ref{@h2_msgs} = (  ); # fetch_hash_2 can select by uid with last arg as ref
 	$h2_fir_ref   =   $imap2->fetch_hash_2("FLAGS", "INTERNALDATE", "RFC822.SIZE", $h2_fir_ref)
 	  if (@h2_msgs);
-	$debug and print "Time fir: ", timenext(), " s\n";
+	$debug and print "Host2 getting flags idate and sizes of folder [$h2_fold] took ", timenext(), " s\n" ;
 	
 	my @h2_msgs_duplicate;
 	foreach my $m (@h2_msgs_no_cache) {
-		my $rc = parse_header_msg($imap2, $m, $h2_heads_ref, $h2_fir_ref, "T", \%h2_hash);
+		my $rc = parse_header_msg($imap2, $m, $h2_heads_ref, $h2_fir_ref, 'Host2', \%h2_hash);
 		my $h2_size = $h2_fir_ref->{$m}->{"RFC822.SIZE"} || 0;
 		if (! defined($rc)) {
-			print "+ Skipping msg #$m:$h2_size in host2 folder $h2_fold (no header so we ignore this message)\n";
+                        print "Host2 $h2_fold/$m size $h2_size ignored (no header so we ignore this message)\n";
 			$h2_nb_msg_noheader += 1 ;
 		} elsif(0 == $rc) {
 			# duplicate
@@ -1152,7 +1187,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 			push(@h2_msgs_duplicate, $m);
 		}
 	}
-	$debug and print "Time parsing headers on host2: ", timenext(), " s\n";
+	$debug and print "Host2 whole time parsing headers took ", timenext(), " s\n";
 
 	$debug and print "++++ Verifying [$h1_fold] -> [$h2_fold]\n";
 	# messages in host1 that are not in host2
@@ -1227,7 +1262,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 		else{
 		        # already on host2
 			my $h2_msg   = $h2_hash{$m_id}{'m'} ;
-			$debug and print "msg $h1_fold/$h1_msg equals $h2_fold/$h2_msg\n" ;
+			$debug and print "msg   $h1_fold/$h1_msg equals $h2_fold/$h2_msg\n" ;
 			$total_bytes_skipped += $h1_size ;
 			$nb_msg_skipped += 1 ;
 			$debugcache and print "touch $cache_dir/${h1_msg}_$h2_msg\n"  if ( $usecache ) ;
@@ -1242,9 +1277,9 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 	    	# Good
 		my $h2_size = $h2_hash{$m_id}{'s'};
 		$debug and print
-		"msg $h1_fold/$h1_msg sizes $h1_size <> $h2_size $h2_fold/$h2_msg\n";
+		"msg   $h1_fold/$h1_msg sizes $h1_size <> $h2_size $h2_fold/$h2_msg\n";
 		if( $delete ) {
-			print "msg $h1_fold/$h1_msg deleted on host1\n";
+			print "msg   $h1_fold/$h1_msg deleted on host1\n";
 			unless( $dry ) {
 				$imap1->delete_message( $h1_msg );
 				$h1_nb_msg_deleted += 1;
@@ -1256,7 +1291,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) {
 	# END MESS: loop
 	MESS_IN_CACHE: foreach my $h1_msg ( @h1_msgs_in_cache ) {
 		my $h2_msg = $cache_1_2_ref->{ $h1_msg } ;
-		$debugcache and print "cache messages update $h1_msg->$h2_msg\n";
+		$debugcache and print "cache messages update flags $h1_msg->$h2_msg\n";
 		sync_flags( $h1_fold, $h1_msg, $h2_fold, $h2_msg, $permanentflags2, $h1_fir_ref, $h2_fir_ref ) ;
 		my $h1_size = $h1_fir_ref->{ $h1_msg }->{ 'RFC822.SIZE' } ;
 		$total_bytes_skipped += $h1_size;
@@ -1297,7 +1332,7 @@ sub size_filtered_flag {
 
 sub sync_flags {
 	my ( $h1_fold, $h1_msg, $h2_fold, $h2_msg, $permanentflags2, $h1_fir_ref, $h2_fir_ref ) = @_ ;
-	$debug and print "sync flags $h1_fold/$h1_msg->$h2_fold/$h2_msg\n";
+	$debug and print "flags $h1_fold/$h1_msg -> $h2_fold/$h2_msg\n";
 
 	my $h1_size = $h1_fir_ref->{$h1_msg}->{"RFC822.SIZE"} ;
 	return() if size_filtered_flag( $h1_size ) ;
@@ -1498,6 +1533,7 @@ sub myconnect {
 	
 	if ($self->Tls) {
 	    starttls($self);
+	    $self->Starttls( 1 ) ;
 	}
 	
 	$self->Ignoresizeerrors($allowsizemismatch);
@@ -1727,8 +1763,8 @@ sub banner_imapsync {
 	my @argv_copy = @_;
 	my $banner_imapsync = join("", 
 		  '$RCSfile: imapsync,v $ ',
-		  '$Revision: 1.422 $ ',
+		  '$Revision: 1.434 $ ',
-		  '$Date: 2011/05/08 17:21:38 $ ',
+		  '$Date: 2011/05/16 07:16:19 $ ',
 		  "\n",localhost_info(), "\n",
 		  "Command line used:\n",
 		  "$0 ", command_line_nopassword(@argv_copy), "\n",
@@ -1799,17 +1835,16 @@ sub select_folder {
 sub create_folder {
 	my( $imap2, $h2_fold, $h1_fold ) = @_ ;
 	
-	if ( $imap2->exists( $h2_fold ) ) {
+	print "Creating folder [$h2_fold] on host2\n";
-		print "Folder $h2_fold already exists on host2.\n";
+        if ( ( 'INBOX' eq uc( $h2_fold) )
-		return( 1 ) ;
+         and ( $imap2->exists( $h2_fold ) ) ) {
-	}else{
+                print "Folder [$h2_fold] already exists\n" ;
-		print "Folder $h2_fold does not exist on host2.\n";
+                return( 1 ) ;
-	}
+        }
-	print "Creating folder [$h2_fold] on host2.\n";
 	if ( ! $dry ){
 		if ( ! $imap2->create($h2_fold)){
-			warn "Couldn't create folder [$h2_fold] from [$h1_fold]: ",
+			warn( "Couldn't create folder [$h2_fold] from [$h1_fold]: ",
-			$imap2->LastError,"\n";
+			$imap2->LastError(), "\n" );
 			$nb_errors++;
 			return(0);
 		}else{
@@ -2123,10 +2158,15 @@ sub foldersizes {
 		my $stot = 0;
 		my $smess = 0;
 		printf("$side folder %-35s", "[$folder]");
-		unless($imap->exists($folder)) {
+                if ( 'Host2' eq $side and ! exists( $h2_folders_all{ $folder } ) ) {
-			print("does not exist yet\n");
+		        print(" does not exist yet\n") ;
-			next;
+			next ;
 		}
+                if ( 'Host1' eq $side and ! exists( $h1_folders_all{ $folder } ) ) {
+		        print(" does not exist\n") ;
+			next ;
+		}
+                
 		unless ($imap->examine($folder)) {
 			warn 
 			  "$side Folder $folder: Could not examine: ",
@@ -2410,13 +2450,21 @@ sub tests_ucsecond {
 }
 
 sub select_msgs {
-	my ($imap) = @_;
+	my ( $imap, $msgs_all_hash_ref ) = @_ ;
-	my (@msgs,@max,@min,@union,@inter);
+	my ( @msgs, @msgs_all, @max, @min, @union, @inter ) ;
 	
-	unless (defined($maxage) or defined($minage)) {
+        
-		#@msgs = $imap->search("ALL");
+	if ( defined( $msgs_all_hash_ref ) 
-		@msgs = $imap->messages();
+        or ( ! defined( $maxage ) and ! defined( $minage ) )
-		return(@msgs);
+        ) {
+		@msgs     = $imap->messages() ;
+                if ( defined( $msgs_all_hash_ref ) ) {
+                        @msgs_all     = @msgs ;
+                        @{ $msgs_all_hash_ref }{ @msgs_all } =  () ;
+                }
+                if ( ! defined( $maxage ) and ! defined( $minage ) ) {
+                        return( @msgs ) ;
+                }
 	}
 	if (defined($maxage)) {
 		@max = $imap->sentsince(time - 86400 * $maxage);
@@ -2500,7 +2548,7 @@ sub copy_message {
 	return() if size_filtered( $h1_size, $h1_msg, $h1_fold, $h2_fold  ) ;
 	
 	my $string;
-	#print "SLEEP 5\n" and sleep 5 ;
+	do { print "SLEEP 5\n" and sleep 5 ; } if ( $debugsleep ) ;
 	print "- msg $h1_fold/$h1_msg S[$h1_size] F[$h1_flags] I[$h1_idate] has RFC822.SIZE null!\n" if ( ! $h1_size ) ;
 
 	
@@ -2678,7 +2726,7 @@ sub tests_cache_map {
 
 sub get_cache {
 	$debugcache and print "Entering get_cache\n";
-	my ($cache_dir, $h1_msgs_ref, $h2_msgs_ref) = @_;
+	my ( $cache_dir, $h1_msgs_ref, $h2_msgs_ref, $h1_msgs_all_hash_ref, $h2_msgs_all_hash_ref ) = @_;
 
 	-d $cache_dir or return( undef ); # exit if cache directory doesn't exist
 	$debugcache and print "cache_dir: $cache_dir\n";
@@ -2690,11 +2738,7 @@ sub get_cache {
 	my( $cache_1_2_ref, $cache_2_1_ref ) 
 	  = cache_map( \@cache_files, $h1_msgs_ref, $h2_msgs_ref ) ;
 	
-	clean_cache( \@cache_files, $cache_1_2_ref ) 
+	clean_cache( \@cache_files, $cache_1_2_ref, $h1_msgs_all_hash_ref, $h2_msgs_all_hash_ref ) ;
-		if ( ! (  defined( $maxsize ) 
-		       or defined( $minsize )
-		       or defined( $maxage  )
-		       or defined( $minage ) ) );
 	
 	#print "\n", map { "c12 $_ -> $cache_1_2_ref->{ $_ }\n" } keys %$cache_1_2_ref ;
 	#print "\n", map { "c21 $_ -> $cache_2_1_ref->{ $_ }\n" } keys %$cache_2_1_ref ;
@@ -2721,16 +2765,19 @@ sub tests_get_cache {
 	tmp/cache/F1/F2/177_777
 	tmp/cache/F1/F2/155_255
 	) ) ;
-	ok( touch(@test_files_cache), 'get_cache: touch tmp/cache/F1/F2/...' ) ;
+	ok( touch( @test_files_cache ), 'get_cache: touch tmp/cache/F1/F2/...' ) ;
 	
 	
 	# on cache: 100_200 101_201 142_242 143_243 177_277 177_377 177_777 155_255
 	# on live:
 	my $msgs_1 = [120, 142, 143, 144,          177      ];
 	my $msgs_2 = [     242, 243,     299, 377, 777, 255 ];
+        
+        my $msgs_all_1 = { 120 => '', 142 => '', 143 => '', 144 => '', 177 => '' } ;
+        my $msgs_all_2 = { 242 => '', 243 => '', 299 => '', 377 => '', 777 => '', 255 => '' } ;
 	
 	my( $c12, $c21 ) ;
-	ok( ( $c12, $c21 ) = get_cache('tmp/cache/F1/F2', $msgs_1, $msgs_2), 'get_cache: 02' );
+	ok( ( $c12, $c21 ) = get_cache( 'tmp/cache/F1/F2', $msgs_1, $msgs_2, $msgs_all_1, $msgs_all_2 ), 'get_cache: 02' );
 	my $a1 = [ sort { $a <=> $b } keys %$c12 ] ;
 	my $a2 = [ sort { $a <=> $b } keys %$c21 ] ;
 	ok( 0 == compare_lists( [ 142, 143, 177 ], $a1 ), 'get_cache: 03' );
@@ -2740,14 +2787,14 @@ sub tests_get_cache {
 	ok( ! -f 'tmp/cache/F1/F2/100_200', 'get_cache: file removed 100_200');
 	ok( ! -f 'tmp/cache/F1/F2/101_201', 'get_cache: file removed 101_201');
 	
-	# test clean_cache not executed
+	# test clean_cache executed
 	$maxage = 2 ;
 	ok( touch(@test_files_cache), 'get_cache: touch tmp/cache/F1/F2/...' ) ;
-	ok( ( $c12, $c21 ) = get_cache('tmp/cache/F1/F2', $msgs_1, $msgs_2), 'get_cache: 02' );
+	ok( ( $c12, $c21 ) = get_cache('tmp/cache/F1/F2', $msgs_1, $msgs_2, $msgs_all_1, $msgs_all_2 ), 'get_cache: 02' );
 	ok( -f 'tmp/cache/F1/F2/142_242', 'get_cache: file kept 142_242');
 	ok( -f 'tmp/cache/F1/F2/142_242', 'get_cache: file kept 143_243');
-	ok( -f 'tmp/cache/F1/F2/100_200', 'get_cache: file NOT removed 100_200');
+	ok( ! -f 'tmp/cache/F1/F2/100_200', 'get_cache: file NOT removed 100_200');
-	ok( -f 'tmp/cache/F1/F2/101_201', 'get_cache: file NOT removed 101_201');
+	ok( ! -f 'tmp/cache/F1/F2/101_201', 'get_cache: file NOT removed 101_201');
 	
 	
 	# strange files
@@ -2774,7 +2821,10 @@ sub tests_get_cache {
 	$msgs_1 = [120, 142, 143, 144,  	177	 ];
 	$msgs_2 = [	242, 243,     299, 377, 777, 255 ];
 	
-	ok( ( $c12, $c21 ) = get_cache('tmp/cache/rr\uee', $msgs_1, $msgs_2), 'get_cache: strange path 02' );
+        $msgs_all_1 = { 120 => '', 142 => '', 143 => '', 144 => '', 177 => '' } ;
+        $msgs_all_2 = { 242 => '', 243 => '', 299 => '', 377 => '', 777 => '', 255 => '' } ;
+        
+	ok( ( $c12, $c21 ) = get_cache('tmp/cache/rr\uee', $msgs_1, $msgs_2, $msgs_all_1, $msgs_all_2), 'get_cache: strange path 02' );
 	$a1 = [ sort { $a <=> $b } keys %$c12 ] ;
 	$a2 = [ sort { $a <=> $b } keys %$c21 ] ;
 	ok( 0 == compare_lists( [ 142, 143, 177 ], $a1 ), 'get_cache: strange path 03' );
@@ -2833,8 +2883,7 @@ sub tests_match_a_cache_file {
 }
 
 sub clean_cache {
-	my $cache_files_ref = shift ;
+	my ( $cache_files_ref, $cache_1_2_ref, $h1_msgs_all_hash_ref, $h2_msgs_all_hash_ref )  = @_ ;
-	my $cache_1_2_ref   = shift ;
 	
 	$debugcache and print "Entering clean_cache\n";
 	
@@ -2843,10 +2892,13 @@ sub clean_cache {
 		$debugcache and print "$file\n" ;
 		my ( $uid1, $uid2 ) = match_a_cache_file( $file ) ;
 		$debugcache and print "u1: $uid1 u2: $uid2 c12: ", $cache_1_2_ref->{ $uid1 } || '', "\n" ;
-		if (  ( ! defined( $uid1 ) )
+#		  or ( ! exists( $cache_1_2_ref->{ $uid1 } ) )
+#		  or ( ! ( $uid2 == $cache_1_2_ref->{ $uid1 } ) )
+		if ( ( ! defined( $uid1 ) )
 		  or ( ! defined( $uid2 ) )
-		  or ( ! exists( $cache_1_2_ref->{ $uid1 } ) )
+                  or ( ! exists( $h1_msgs_all_hash_ref->{ $uid1 } ) )
-		  or ( ! ( $uid2 == $cache_1_2_ref->{ $uid1 } ) ) ) {
+                  or ( ! exists( $h2_msgs_all_hash_ref->{ $uid2 } ) )
+                ) {
 			$debugcache and print "remove $file\n" ;
 			unlink( $file ) or warn "$!" ;
 		}
@@ -2885,8 +2937,18 @@ sub tests_clean_cache {
 		142 => 242,
 		177 => 777,
 	} ;
-	
+        
-	ok( clean_cache( \@test_files_cache, $cache ), 'clean_cache: ' ) ;
+        my $all_1 = {
+                142 => '',
+                177 => '',
+        } ;
+        
+        my $all_2 = {
+                200 => '',
+                242 => '',
+                777 => '',
+        } ;
+	ok( clean_cache( \@test_files_cache, $cache, $all_1, $all_2 ), 'clean_cache: ' ) ;
 	
 	ok( ! -f 'tmp/cache/G1/G2/100_200', 'clean_cache: 100_200 after' );
 	ok(   -f 'tmp/cache/G1/G2/142_242', 'clean_cache: 142_242 after' );
@@ -2895,7 +2957,61 @@ sub tests_clean_cache {
 	ok(   -f 'tmp/cache/G1/G2/177_777', 'clean_cache: 177_777 after' );
 	ok( ! -f 'tmp/cache/G1/G2/155_255', 'clean_cache: 155_255 after' );
 }
+
+sub tests_clean_cache_2 {
+
+	ok( ( ! -d  'tmp/cache/G1/G2' or rmtree( 'tmp/cache/G1/G2' )), 'clean_cache_2: rmtree tmp/cache/G1/G2' ) ;
+	ok( mkpath( 'tmp/cache/G1/G2' ), 'clean_cache_2: mkpath tmp/cache/G1/G2' ) ;
 	
+	my @test_files_cache = ( qw( 
+	tmp/cache/G1/G2/100_200
+	tmp/cache/G1/G2/101_201
+	tmp/cache/G1/G2/120_220
+	tmp/cache/G1/G2/142_242
+	tmp/cache/G1/G2/143_243
+	tmp/cache/G1/G2/177_277
+	tmp/cache/G1/G2/177_377
+	tmp/cache/G1/G2/177_777
+	tmp/cache/G1/G2/155_255
+	) ) ;
+	ok( touch(@test_files_cache), 'clean_cache_2: touch tmp/cache/G1/G2/...' ) ;
+	
+	ok( -f 'tmp/cache/G1/G2/100_200', 'clean_cache_2: 100_200 before' );
+	ok( -f 'tmp/cache/G1/G2/142_242', 'clean_cache_2: 142_242 before' );
+	ok( -f 'tmp/cache/G1/G2/177_277', 'clean_cache_2: 177_277 before' );
+	ok( -f 'tmp/cache/G1/G2/177_377', 'clean_cache_2: 177_377 before' );
+	ok( -f 'tmp/cache/G1/G2/177_777', 'clean_cache_2: 177_777 before' );
+	ok( -f 'tmp/cache/G1/G2/155_255', 'clean_cache_2: 155_255 before' );
+	
+	my $cache = {
+		142 => 242,
+		177 => 777,
+	} ;
+        
+        my $all_1 = {
+                100 => '',
+                142 => '',
+                177 => '',
+        } ;
+        
+        my $all_2 = {
+                200 => '',
+                242 => '',
+                777 => '',
+        } ;
+        
+        
+        
+	ok( clean_cache( \@test_files_cache, $cache, $all_1, $all_2 ), 'clean_cache_2: ' ) ;
+	
+	ok(   -f 'tmp/cache/G1/G2/100_200', 'clean_cache_2: 100_200 after' );
+	ok(   -f 'tmp/cache/G1/G2/142_242', 'clean_cache_2: 142_242 after' );
+	ok( ! -f 'tmp/cache/G1/G2/177_277', 'clean_cache_2: 177_277 after' );
+	ok( ! -f 'tmp/cache/G1/G2/177_377', 'clean_cache_2: 177_377 after' );
+	ok(   -f 'tmp/cache/G1/G2/177_777', 'clean_cache_2: 177_777 after' );
+	ok( ! -f 'tmp/cache/G1/G2/155_255', 'clean_cache_2: 155_255 after' );
+}
+
 
 sub tests_touch {
 
@@ -2908,6 +3024,14 @@ sub tests_touch {
 
 }
 
+sub tests_mkpath {
+
+        my $long_path = "123456789/" x 30 ;
+	ok( (-d "tmp/tests/long/$long_path" or  mkpath( "tmp/tests/long/$long_path" )), 'tests_mkpath: mkpath > 300 char' ) ;
+	ok( (-d "tmp/tests/long/$long_path" and rmtree( "tmp/tests/long/" )),           'tests_mkpath: rmtree > 300 char' ) ;
+        ok( 1 == 1, 'tests_mkpath: 1 == 1' ) ;
+}
+
 sub touch {
 	my @files = @_ ;
 	my @result;
@@ -3101,6 +3225,7 @@ sub get_options {
                                    "debug!"        => \$debug,
                                    "debugLIST!"    => \$debugLIST,
                                    "debugcontent!" => \$debugcontent,
+				   "debugsleep!"   => \$debugsleep,
                                    "debugflags!"   => \$debugflags,
                                    "debugimap!"    => \$debugimap,
                                    "debugimap1!"   => \$debugimap1,
@@ -3163,7 +3288,7 @@ sub get_options {
                                    "timeout=i"   => \$timeout,
 				   "skipheader=s" => \$skipheader,
 				   "useheader=s" => \@useheader,
-				   "takebody!"   => \$takebody,
+				   "wholeheaderifneeded!"   => \$wholeheaderifneeded,
 				   "skipsize!"   => \$skipsize,
 				   "allowsizemismatch!" => \$allowsizemismatch,
 				   "fastio1!"     => \$fastio1,
@@ -3247,11 +3372,30 @@ sub parse_header_msg {
 	
 	my $head = $s_heads->{$m_uid};
 	my $headnum =  scalar(keys(%$head));
-	$debug and print "Head NUM:", $headnum, "\n";
+	$debug and print "$s uid $m_uid head nb pass one: ", $headnum, "\n";
-	unless($headnum) { print "Warning: no header used or found for message $m_uid\n"; }
+	
 	my $headstr;
 	
+	if ( ( ! $headnum ) and ( $wholeheaderifneeded ) ){
+		print "$s uid $m_uid no header by classic way so taking whole header\n";
+		$imap->fetch($m_uid, "BODY.PEEK[HEADER]");
+		my $whole_header = $imap->_transaction_literals;
+		
+                #print $whole_header;
+                $head = decompose_header( $whole_header ) ;
+                
+                $headnum =  scalar( keys( %$head ) ) ;
+	        $debug and print "$s uid $m_uid head nb pass two: ", $headnum, "\n";
+	}
+
+        #require Data::Dumper ;
+        #print Data::Dumper->Dump( [ $head, \%useheader ] ) ;
+
 	foreach my $h (sort keys(%$head)){
+                next if ( ! exists( $useheader{ uc( $h ) } )
+                      and ! exists( $useheader{ 'ALL' } ) 
+                      and ! exists( $useheaderclassic{ uc( $h ) } ) 
+                ) ;
 		foreach my $val (sort @{$head->{$h}}) {
 			# no 8-bit data in headers !
 			$val =~ s/[\x80-\xff]/X/g;
@@ -3264,29 +3408,16 @@ sub parse_header_msg {
 			
 			my $H = uc("$h: $val");
 			# show stuff in debug mode
-			$debug and print "${s}H $H", "\n";
+			$debug and print "$s uid $m_uid header [$H]", "\n";
 			
 			if ($skipheader and $H =~ m/$skipheader/i) {
-				$debug and print "Skipping header $H\n";
+				$debug and print "$s uid $m_uid skipping header [$H]\n";
 				next;
 			}
 			$headstr .= "$H";
 		}
 	}
 	
-	if ( ( ! $headstr) and ( $takebody ) ){
-		print "no header so taking body first 2Ko\n";
-		$imap->fetch($m_uid, "BODY.PEEK[TEXT]<0.2048>");
-		$headstr = $imap->_transaction_literals;
-		
-		if ( 4048 <= length( $headstr ) ) {
-		# the imap server might reply the whole message
-		# this is bad for memory on huge mailboxes
-			$takebody = 0 ;
-			$headstr = '' ;
-			$h1_msgs_copy_by_uid{ $m_uid } = 1 ;
-		}
-	}
 	return() if ( ! $headstr );
 	
 	my $size  = $s_fir->{$m_uid}->{"RFC822.SIZE"};
@@ -3294,7 +3425,7 @@ sub parse_header_msg {
 	my $idate = $s_fir->{$m_uid}->{"INTERNALDATE"};
 	$size = length($headstr) unless ($size);
 	my $m_md5 = md5_base64($headstr);	
-	$debug and print "$s msg $m_uid:$m_md5:$size\n";
+	$debug and print "$s uid $m_uid sig $m_md5 size $size\n";
 	my $key;
         if ($skipsize) {
                 $key = "$m_md5";
@@ -3375,7 +3506,7 @@ sub check_last_release {
 }
 
 sub imapsync_version {
-	my $rcs = '$Id: imapsync,v 1.422 2011/05/08 17:21:38 gilles Exp gilles $ ';
+	my $rcs = '$Id: imapsync,v 1.434 2011/05/16 07:16:19 gilles Exp gilles $ ';
 	$rcs =~ m/,v (\d+\.\d+)/;
 	my $VERSION = ($1) ? $1: "UNKNOWN";
 	return($VERSION);
@@ -3678,7 +3809,7 @@ sub good_date {
     	#print "internal: [$1][$2][$3][$4]\n";
 	my ($day_1, $date_rest, $hour, $zone) = ($1,$2,$3,$4);
 	$day_1 = '0' if ($day_1 eq '');
-	$zone  = ''  if not defined($zone);
+	$zone  = ' +0000'  if not defined($zone);
 	$d = $day_1 . $date_rest . $hour . $zone;
 	
 	
@@ -3686,7 +3817,7 @@ sub good_date {
     	#print "header: [$1][$2][$3][$4][$5][$6]\n";
 	my ($day_1, $day_rest, $month, $year, $hour, $zone) = ($1,$2,$3,$4,$5,$6);
 	$day_1 = '0' if ($day_1 eq '');
-	$zone  = ''  if not defined($zone);
+	$zone  = ' +0000'  if not defined($zone);
 	$d = $day_1 . "$day_rest-$month-$year" . $hour . $zone;
 	
     }else{
@@ -3811,10 +3942,10 @@ sub tests_good_date {
 
 	ok('' eq good_date(), 'good_date no arg');
 	ok('"24-Aug-2010 16:00:00 +0200"' eq good_date('24-Aug-2010 16:00:00 +0200'), 'good_date internal 2digit zone');
-	ok('"24-Aug-2010 16:00:00"'       eq good_date('24-Aug-2010 16:00:00'), 'good_date internal 2digit no zone');
+	ok('"24-Aug-2010 16:00:00 +0000"' eq good_date('24-Aug-2010 16:00:00'), 'good_date internal 2digit no zone');
 	ok('"01-Sep-2010 16:00:00 +0200"' eq good_date( '1-Sep-2010 16:00:00 +0200'), 'good_date internal SP 1digit');
 	ok('"24-Aug-2010 16:00:00 +0200"' eq good_date('Tue, 24 Aug 2010 16:00:00 +0200'), 'good_date header 2digit zone');
-	ok('"01-Sep-2010 16:00:00"'       eq good_date('Wed, 1 Sep 2010 16:00:00'), 'good_date header SP 1digit zone');
+	ok('"01-Sep-2010 16:00:00 +0000"' eq good_date('Wed, 1 Sep 2010 16:00:00'), 'good_date header SP 1digit zone');
 	ok('"01-Sep-2010 16:00:00 +0200"' eq good_date('Wed, 1 Sep 2010 16:00:00 +0200'), 'good_date header SP 1digit zone');
 	ok('"01-Sep-2010 16:00:00 +0200"' eq good_date('Wed, 1 Sep 2010 16:00:00 +0200 (CEST)'), 'good_date header SP 1digit zone');
 
@@ -3881,11 +4012,132 @@ sub delete_folders_in_2_not_in_1 {
 	}
 }
 
+
+sub extract_header {
+        my $string = shift ;
+        
+        my ( $header ) = split( /\n\n/, $string ) ;
+        #print "[$header]\n" ;
+        return( $header ) ;
+}
+
+sub tests_extract_header {
+        ok( 
+'Message-Id: <20100428101817.A66CB162474E@plume.est.belle>
+Date: Wed, 28 Apr 2010 12:18:17 +0200 (CEST)
+From: gilles@louloutte.dyndns.org (Gilles LAMIRAL)' 
+eq extract_header(
+'Message-Id: <20100428101817.A66CB162474E@plume.est.belle>
+Date: Wed, 28 Apr 2010 12:18:17 +0200 (CEST)
+From: gilles@louloutte.dyndns.org (Gilles LAMIRAL)
+
+body
+lalala
+' ), 'extract_header: 1') ;
+     
+}
+
+sub decompose_header{
+        my $string = shift ;
+        
+        my $header = {  } ;
+        
+        my ($key, $val ) ;
+        my @line = split( /\n|\r\n/, $string ) ;
+        foreach my $line ( @line ) {
+                #print "DDD $line\n" ;
+                if( $line =~ m/(^[^     :]+): (.*)/ ) {
+                        $key = $1 ;
+                        $val = $2 ;
+                        #print "DDD [$key] [$val]\n" ;
+                        push( @{ $header->{ $key } }, $val ) ;
+                }elsif( $line =~ m/^(\s+)(.*)/ ) {
+                        $val = $2 ;
+                        #print "DDD only [$val]\n" ;
+                        @{ $header->{ $key } }[ -1 ] .= " $val" ;
+                }else{
+                        next ;
+                }
+        }
+        #require Data::Dumper ;
+        #print Data::Dumper->Dump( [ $header ] ) ;
+
+        return( $header ) ;
+}
+
+
+sub tests_decompose_header{
+
+        my $header_dec ;
+        
+        $header_dec = decompose_header(
+'KEY_1: VAL_1
+KEY_2: VAL_2
+  VAL_2_+
+        VAL_2_++
+KEY_3: VAL_3
+KEY_1: VAL_1_other
+'
+        ) ;
+        
+        ok( 'VAL_3' 
+        eq $header_dec->{ 'KEY_3' }[0], 'decompose_header: VAL_3' ) ;
+
+        ok( 'VAL_1' 
+        eq $header_dec->{ 'KEY_1' }[0], 'decompose_header: VAL_1' ) ;
+
+        ok( 'VAL_1_other' 
+        eq $header_dec->{ 'KEY_1' }[1], 'decompose_header: VAL_1_other' ) ;
+
+        ok( 'VAL_2 VAL_2_+ VAL_2_++' 
+        eq $header_dec->{ 'KEY_2' }[0], 'decompose_header: VAL_2 VAL_2_+ VAL_2_++' ) ;
+
+        
+        $header_dec = decompose_header(
+'Message-Id: <20100428101817.A66CB162474E@plume.est.belle>
+Date: Wed, 28 Apr 2010 12:18:17 +0200 (CEST)
+From: gilles@louloutte.dyndns.org (Gilles LAMIRAL)'
+        ) ;
+
+        ok( '<20100428101817.A66CB162474E@plume.est.belle>' 
+        eq $header_dec->{ 'Message-Id' }[0], 'decompose_header: 1' ) ;
+
+        $header_dec = decompose_header(
+'Return-Path: <gilles@louloutte.dyndns.org>
+Received: by plume.est.belle (Postfix, from userid 1000)
+        id 120A71624742; Wed, 28 Apr 2010 01:46:40 +0200 (CEST)
+Subject: test:eekahceishukohpe'
+        ) ;
+        ok( 
+'by plume.est.belle (Postfix, from userid 1000) id 120A71624742; Wed, 28 Apr 2010 01:46:40 +0200 (CEST)' 
+        eq $header_dec->{ 'Received' }[0], 'decompose_header: 2' ) ;
+
+        $header_dec = decompose_header(
+'Received: from plume (localhost [127.0.0.1])
+        by plume.est.belle (Postfix) with ESMTP id C6EB73F6C9
+        for <gilles@localhost>; Mon, 26 Nov 2007 10:39:06 +0100 (CET)
+Received: from plume [192.168.68.7]
+        by plume with POP3 (fetchmail-6.3.6)
+        for <gilles@localhost> (single-drop); Mon, 26 Nov 2007 10:39:06 +0100 (CET)'
+        ) ;
+        ok(
+        'from plume (localhost [127.0.0.1]) by plume.est.belle (Postfix) with ESMTP id C6EB73F6C9 for <gilles@localhost>; Mon, 26 Nov 2007 10:39:06 +0100 (CET)'
+        eq $header_dec->{ 'Received' }[0], 'decompose_header: 3' ) ;
+        ok(
+        'from plume [192.168.68.7] by plume with POP3 (fetchmail-6.3.6) for <gilles@localhost> (single-drop); Mon, 26 Nov 2007 10:39:06 +0100 (CET)'
+        eq $header_dec->{ 'Received' }[1], 'decompose_header: 3' ) ;
+}
+
+
 sub tests_debug {
 	
       SKIP: {
 		skip "No test in normal run" if ( not $tests_debug );
-		tests_flagsCase(  ) ;
+		tests_match_a_cache_file(  ) ;
+		tests_cache_map(  ) ;
+		tests_get_cache(  ) ;
+		tests_clean_cache(  ) ;
+		tests_clean_cache_2(  ) ;
 	}
 }
 
@@ -3909,13 +4161,17 @@ sub tests {
 		tests_imapsync_basename();
 		tests_list_keys_in_2_not_in_1();
 		tests_convert_sep_to_slash(  ) ;
+		tests_match_a_cache_file(  ) ;
 		tests_cache_map(  ) ;
 		tests_get_cache(  ) ;
 		tests_clean_cache(  ) ;
-		tests_match_a_cache_file(  ) ;
+		tests_clean_cache_2(  ) ;
 		tests_touch(  ) ;
 		tests_ucsecond(  ) ;
-		
+		tests_flagsCase(  ) ;
+                tests_mkpath(  ) ;
+                tests_extract_header(  ) ;
+                tests_decompose_header(  ) ;
 	}
 }
 
@@ -4121,7 +4377,7 @@ use constant NonFolderArg => 1;                 # Value to pass to Massage to
 	my $headers = {};       # hash from message ids to header hash
 	my $split = $self->Split() || scalar(@$msgspec_all);
 	while(my @msgs = splice(@$msgspec_all, 0, $split)) {
-		$debug and print "SPLIT: @msgs\n";
+		#$debug and print "SPLIT: @msgs\n";
 		my $msgspec = \@msgs;
 
         # Make $msg a comma separated list, of messages we want
@@ -4384,7 +4640,18 @@ no warnings 'once';
 	return $self->{SSL};
 };
 
+*Mail::IMAPClient::Starttls = sub {
+	my $self = shift;
+	
+	if (@_) { $self->{Starttls} = shift }
+	return $self->{Starttls};
+};
+
+
+
 *Mail::IMAPClient::exists = sub  {
+    # Bad implementation STATUS fails and can close the connexion
+    # Exchange does this after 10 failures
     my ( $self, $folder ) = @_;
     $self->status($folder) ? $self : undef;
 };

index.shtml

@@ -5,7 +5,7 @@
 <title>Imapsync: an IMAP migration tool ( release <!--#exec cmd="cat VERSION"--> )</title>
 <meta name="generator" content="Bluefish 1.0.7"/>
 <meta name="author" content="Gilles LAMIRAL"/>
-<meta name="date" content="2011-05-09T02:42:11+0200"/>
+<meta name="date" content="2011-05-16T19:09:12+0200"/>
 <meta name="copyright" content="None"/>
 <meta name="keywords" content="imap, transfert, migration"/>
 <meta name="description" content="imap migration tool"/>
@@ -71,6 +71,20 @@ where the user plays independently on both sides. Use <b>offlineimap</b>
 <p>New features or bugfixes since previous release 1.411:</p>
 
 <ul>
+<li><b>1.434</b></li>
+<li><b>Bugfix</b>: Changed the way imapsync knows <b>whether a folder exists</b> or not. <b>Exchange</b> might be happy and <b>stop deconnecting</b> for this reason.</li>
+<li><b>Bugfix</b>: <b>Reconnections</b> are well done in <b>TLS mode</b> now.</li>
+<li><b>Bugfix</b>:  IMAP RFC 3501 and some imap servers require <b>internal dates</b> have a <b>zone data</b>. Default <b>to +0000</b> when host1 doesn't give it.</li>
+<li><b>Bugfix</b>: Options --maxsize --minsize now really work with --useuid (1.422 had a cache issue).</li>
+<li><b>Bugfix</b>: <b>Improved</b> the way imapsync <b>deals with headers</b>:<ul>
+	<li><b>Stopped</b> getting <b>first 2KB body</b> of message. Not a good idea in practice.</li>
+	<li>If <b>massive</b> get headers <b>fails</b> then take the <b>whole header</b> one by one (instead of body).</li>
+	<li>Default is like --useheader <b>"Message-Id"</b> --useheader <b>"Message-ID"</b> (instead of just "Message-Id").</li>
+	<li>Use header "Message-Id" and header "Date" as md5 signature when taking the whole header.</li>
+</ul>
+</li>
+
+<li><b>1.422</b></li>
 <li><b>Better default behavior</b>: Option --delete2 implies --expunge2 now (unless --noexpunge2 is given.)</li>
 <li><b>Better default behavior</b>: Correct flags case to be RFC compliant on host2 if host1 is not (\SEEN -> \Seen)</li>
 <li><b>Better debug</b>: Added --debugcontent option to avoid debugging content (can be big) with --debug option.</li>
@@ -79,7 +93,7 @@ where the user plays independently on both sides. Use <b>offlineimap</b>
 <li><b>Bugfix</b>: Options --maxsize --minsize now work with --useuid</li>
 <li><b>Bugfix</b>: Flag sync of already transfered messages now take care of --maxsize --minsize options</li>
 <li><b>Bugfix</b>: Added 0 length message tracking when fetching host1 (to avoid frequently "APPEND {0}" recent issues).</li>
-<li><b>Bugfix</b>: Avoid now Inbox <-> INBOX problem ("already exists").</li>
+<li><b>Bugfix</b>: Avoid Inbox vs INBOX case problem ("already exists").</li>
 <li><b>Bugfix</b>: --proxyauth2 was setting proxyauth1 instead of proxyauth2</li>
 </ul>
 
@@ -356,23 +370,25 @@ will be to code it or fix it.<br/>
 	<li>Deerfield VisNetic MailServer 5.8.6 [host1]</li>
 	<li>dkimap4 [host1]</li>
 	<li>Domino (Notes) 4.61[host1], 6.5, 5.0.6, 5.0.7, 7.0.2, 6.0.2CF1, 7.0.1[host1], 8.0.1[host1]</li>
-	<li>Dovecot 0.99.10.4, 0.99.14, 0.99.14-8.fc4, 1.0-0.beta2.7, 
+	<li><b>Dovecot</b> 0.99.10.4, 0.99.14, 0.99.14-8.fc4, 1.0-0.beta2.7, 
    1.0.0 [dest/source] (LGPL) (http://www.dovecot.org/)</li>
 	<li>Eudora WorldMail v2</li>
-	<li>Gimap (Gmail imap) [host1] [host2]</li>
+	<li><b>Gimap</b> (<b>Gmail</b> imap) [host1] [host2]</li>
 	<li>GMX IMAP4 StreamProxy.</li>
 	<li>Godaddy IMAP (since Godaddy runs Courier)</li>
 	<li>Groupwise IMAP (Novell) 6.x and 7.0. Buggy so see the FAQ.</li>
 	<li>hMailServer 5.3.3 [host2], 4.4.1 [host1], HMAILSERVER 5.3.2-B1769 on windows 2003 [hsot2]</li>
 	<li>iPlanet Messaging server 4.15, 5.1, 5.2</li>
 	<li>IMail 7.15 (Ipswitch/Win2003), 8.12, 11.03 [host1]</li>
-	<li>MailEnable 4.23 [host1] [host2]</li>
+	<li><b>MailEnable</b> 4.23 [host1][host2], 4.26 [host1][host2]</li>
 	<li>MDaemon 7.0.1, 8.0.2, 8.1, 9.5.4 (Windows server 2003 R2 platform), 12 [host2]</li>
 	<li>Mercury 4.1 (Windows server 2000 platform)</li>
-	<li>Microsoft Exchange Server 5.5, 6.0.6249.0[host1], 6.0.6487.0[host1], 
+	<li><b>Microsoft Exchange Server</b> 5.5, 6.0.6249.0[host1], 6.0.6487.0[host1], 
    6.5.7638.1 [host2], 6.5 [host1], Exchange 2007 SP1 (with Update Rollup 2), 
    Exchange2007-EP-SP2, 
-   Exchange 2010 RTM (Release to Manufacturing) [host2]</li>
+   Exchange 2010 RTM (Release to Manufacturing) [host2],
+   Exchange 2010 SP1 RU2 [host2]
+   </li>
    <li>Mirapoint server</li>
 	<li>Netscape Mail Server 3.6 (Wintel)</li>
 	<li>Netscape Messaging Server 4.15 Patch 7</li>
@@ -392,7 +408,8 @@ will be to code it or fix it.<br/>
    (http://www.washington.edu/imap/)</li>
 	<li>UW - QMail v2.1</li>
 	<li>VMS, Imap part of TCP/IP suite of VMS 7.3.2</li>
-	<li>Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5, 6.x</li>
+	<li><b>Zimbra-IMAP</b> 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 
+               Zimbra 5.0.24_GA_3356.RHEL4 [host1], 5.5, 6.x</li>
 </ul>
 
 <hr/>
@@ -421,7 +438,7 @@ alt="Viewable With Any Browser" />
 <!--#config timefmt="%D" -->
 <!--#config timefmt="%A %B %d, %Y" -->
 <b>This document last modified on <!--#echo var="LAST_MODIFIED" --></b>
-($Id: index.shtml,v 1.66 2011/05/09 00:45:40 gilles Exp gilles $)
+($Id: index.shtml,v 1.69 2011/05/16 17:10:13 gilles Exp gilles $)
 </p>
 
 </body>

paypal_return.shtml

@@ -5,7 +5,7 @@
 <title>imapsync download</title>
 <meta name="generator" content="Bluefish 1.0.7"/>
 <meta name="author" content="Gilles LAMIRAL"/>
-<meta name="date" content="2011-05-07T04:53:07+0200"/>
+<meta name="date" content="2011-05-16T19:22:54+0200"/>
 <meta name="copyright" content=""/>
 <meta name="keywords" content=""/>
 <meta name="description" content=""/>
@@ -31,11 +31,7 @@ border:0px;
 </style>
 </head>
 
-
 <body>
-
-
-
 <h1>imapsync download</h1>
 
 <p><b>I thank you for buying Imapsync!</b></p>
@@ -46,12 +42,12 @@ You may log into your account at <a href="http://www.paypal.com/">www.paypal.com
 to view details of this transaction. 
 </p>
 
-<p>You will find the latest <b>imapsync source code</b> release 1.417 at the following link:<br/>
+<p>You will find the latest <b>imapsync source code</b> release 1.422 at the following link:<br/>
-<a href="http://www.linux-france.org/depot/2011_05_07/8qkE2L/">http://www.linux-france.org/depot/2011_05_07/8qkE2L/</a>
+<a href="http://www.linux-france.org/depot/2011_05_09/EocZFt/">http://www.linux-france.org/depot/2011_05_09/EocZFt/</a>
 </p>
 
-<p>You will find the latest <b>imapsync.exe binary</b> release 1.417 at the following link:<br/>
+<p>You will find the latest <b>imapsync.exe binary</b> release 1.422 at the following link:<br/>
-<a href="http://www.linux-france.org/depot/2011_05_07/eQ5bXu/">http://www.linux-france.org/depot/2011_05_07/eQ5bXu/</a>
+<a href="http://www.linux-france.org/depot/2011_05_09/XhVbYj/">http://www.linux-france.org/depot/2011_05_09/XhVbYj/</a>
 </p>
 
 <p>You will receive an invoice soon.</p>
@@ -59,8 +55,13 @@ to view details of this transaction.
 <p>Next imapsync releases will be available for one year without extra payment.<br/>
 I will send you a message explaining how to get them</p>
 
-<p>I thank you again for buying and using imapsync,
+<p>To <b>avoid loosing time</b>, explain your specific needs, find <b>best solutions</b><br/>
-I wish you successful imap transfers!</p>
+and then <b>succeed your migration</b> you can buy professionnal support at the link<br/>
+<a href="http://www.linux-france.org/prj/imapsync/#buy_support"><b>http://www.linux-france.org/prj/imapsync/#buy_support</b></a>
+</p>
+
+<p><b>I thank you</b> again for buying and using imapsync,<br/>
+<b>I wish you successful imap transfers!</b></p>
 
 <p><a href="./">imapsync homepage</a></p>
 
@@ -82,7 +83,7 @@ gilles.lamiral@laposte.net</p>
 <!--#config timefmt="%D" -->
 <!--#config timefmt="%A %B %d, %Y" -->
 <b>This document last modified on <!--#echo var="LAST_MODIFIED" --></b><br/>
-($Id: paypal_return.shtml,v 1.5 2011/05/07 02:56:35 gilles Exp gilles $)
+($Id: paypal_return.shtml,v 1.7 2011/05/16 17:23:12 gilles Exp gilles $)
 </p>
 
 <!-- Google Code for Achat imapsync Conversion Page -->

test2.bat

@@ -1,5 +1,5 @@
 
-REM $Id: test.bat,v 1.8 2011/01/15 06:30:33 gilles Exp gilles $
+REM $Id: test2.bat,v 1.2 2011/05/16 16:41:47 gilles Exp gilles $
 
 cd C:\msys\1.0\home\Admin\imapsync
 REM perl ./imapsync --host1 p  --user1 tata --passfile1 secret.tata  --host2 p --user2 titi --passfile2 secret.titi --delete2 --expunge2 --folder INBOX 
@@ -9,6 +9,13 @@ REM perl ./imapsync --host1 p  --user1 tata --passfile1 secret.tata  --host2 p -
 
 
 REM imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --justfolders --nofoldersize --folder INBOX.yop.yap --sep1 / --regextrans2 "s,/,_," 
-imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --nofoldersize --folder INBOX.yop.yap --regexflag 's/\\Answered//g' --debug > out.txt  
+REM imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --nofoldersize --folder INBOX.yop.yap --regexflag 's/\\Answered//g' --debug > out.txt  
 
+REM perl imapsync --version
+REM perl imapsync --tests_debug
+
+imapsync.exe ^
+  --host1 p --user1 big1 --passfile1 secret.big1 ^
+  --host2 p --user2 big2 --passfile2 secret.big2 ^
+  --folder INBOX.bigmail
 

tests.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# $Id: tests.sh,v 1.163 2011/05/09 00:10:16 gilles Exp gilles $  
+# $Id: tests.sh,v 1.165 2011/05/16 16:41:11 gilles Exp gilles $  
 
 # Example 1:
 # CMD_PERL='perl -I./Mail-IMAPClient-3.25/lib' sh -x tests.sh
@@ -264,7 +264,7 @@ ll_folder_create_INBOX_Inbox() {
                 --host2 $HOST2 --user2 titi \
                 --passfile2 ../../var/pass/secret.titi \
                 --folder INBOX --regextrans2 's/INBOX/Inbox/' \
-		--justfolders
+		--justfolders --nofoldersizes
 }
 
 
@@ -287,6 +287,7 @@ ll_debugimap() {
                 --passfile2 ../../var/pass/secret.titi \
                 --folder INBOX.oneemail --debugimap
 }
+
 ll_few_emails() {
                 $CMD_PERL ./imapsync \
                 --host1 $HOST1  --user1 tata \
@@ -296,6 +297,38 @@ ll_few_emails() {
                 --folder INBOX.few_emails
 }
 
+ll_noheader() {
+                $CMD_PERL ./imapsync \
+                --host1 $HOST1  --user1 tata \
+                --passfile1 ../../var/pass/secret.tata \
+                --host2 $HOST2 --user2 titi \
+                --passfile2 ../../var/pass/secret.titi \
+                --folder INBOX.few_emails --useheader ''
+}
+
+ll_noheader_force() {
+                $CMD_PERL ./imapsync \
+                --host1 $HOST1  --user1 tata \
+                --passfile1 ../../var/pass/secret.tata \
+                --host2 $HOST2 --user2 titi \
+                --passfile2 ../../var/pass/secret.titi \
+                --folder INBOX.few_emails \
+		--useheader '' \
+		--skipheader 'Message-Id|Date'
+}
+
+
+
+ll_usecachemaxage() {
+                $CMD_PERL ./imapsync \
+                --host1 $HOST1  --user1 tata \
+                --passfile1 ../../var/pass/secret.tata \
+                --host2 $HOST2 --user2 titi \
+                --passfile2 ../../var/pass/secret.titi \
+                --useuid --maxage 3
+}
+
+
 
 ll_folderrec() {
                 $CMD_PERL ./imapsync \
@@ -523,6 +556,19 @@ ll_justfoldersizes()
                 --justfoldersizes
 }
 
+ll_justfoldersizes_noexist() 
+{
+                $CMD_PERL ./imapsync \
+                --host1 $HOST1 --user1 tata \
+                --passfile1 ../../var/pass/secret.tata \
+                --host2 $HOST2 --user2 titi \
+                --passfile2 ../../var/pass/secret.titi \
+                --justfoldersizes --folder NoExist --folder INBOX
+}
+
+
+
+
 ll_dev_reconnect() 
 {
 # in another terminal:
@@ -580,6 +626,35 @@ EOF
 		--delete2
 }
 
+ll_dev_reconnect_tls() 
+{
+# in another terminal:
+#
+: <<'EOF'
+while :; do 
+    killall -v -u vmail imapd; 
+    RAND_WAIT=`numrandom .1..5i.1`
+    echo sleeping $RAND_WAIT
+    sleepenh $RAND_WAIT
+done
+# or 
+while read y; do
+    echo ENTER to kill all imapd
+    killall -v -u vmail imapd; 
+done
+
+EOF
+        can_send && sendtestmessage
+#        can_send && sendtestmessage
+        $CMD_PERL ./imapsync \
+                --host1 $HOST1 --tls1 --user1 tata \
+                --passfile1 ../../var/pass/secret.tata \
+                --host2 $HOST2 --tls2 --user2 titi \
+                --passfile2 ../../var/pass/secret.titi \
+                --folder INBOX --useuid \
+		--delete2 --debugsleep --debugimap
+}
+
 
 
 
@@ -646,7 +721,8 @@ ll_newmessage()
         --passfile1 ../../var/pass/secret.tata \
         --host2 $HOST2 --user2 titi \
         --passfile2 ../../var/pass/secret.titi \
-        --maxage 1 --folder INBOX --nofoldersizes --noreleasecheck 
+        --maxage 1 --folder INBOX --nofoldersizes --noreleasecheck \
+	--debugLIST
 }
 
 ll_folder_INBOX()
@@ -694,9 +770,23 @@ ll_maxsize_useuid()
                 --host2 $HOST2 --user2 titi \
                 --passfile2 ../../var/pass/secret.titi \
                 --maxsize 10 --nofoldersizes --folder INBOX \
-                --useuid
+                --useuid --debugcache
 }
 
+ll_minsize_useuid() 
+{       
+        can_send && sendtestmessage
+        $CMD_PERL ./imapsync \
+                --host1 $HOST1 --user1 tata \
+                --passfile1 ../../var/pass/secret.tata \
+                --host2 $HOST2 --user2 titi \
+                --passfile2 ../../var/pass/secret.titi \
+                --nofoldersizes --folder INBOX \
+                --useuid --debugLIST --minsize 500 --maxage 1
+}
+
+
+
 
 ll_skipsize() 
 {
@@ -2039,6 +2129,8 @@ ll_folder_create_INBOX_Inbox
 ll_delete2folders
 ll_useuid
 ll_useuid_nousecache
+ll_noheader_force
+ll_noheader
 '
 
 other_tests='