1.132

CREDITS

@@ -1,5 +1,8 @@
 #!/bin/cat
 
+Costa Tsaousis
+Had a problem with a huge mailbox (20000 messages)
+
 Scott Glenn
 Found the bug about --regexmess applied only on the first
 message.

ChangeLog

@@ -1,15 +1,27 @@
 
 RCS file: RCS/imapsync,v
 Working file: imapsync
-head: 1.129
+head: 1.132
 branch:
 locks: strict
 access list:
 symbolic names:
 keyword substitution: kv
-total revisions: 129;	selected revisions: 129
+total revisions: 132;	selected revisions: 132
 description:
 ----------------------------
+revision 1.132
+date: 2005/05/20 03:18:48;  author: gilles;  state: Exp;  lines: +23 -16
+Updated doc with all options.
+----------------------------
+revision 1.131
+date: 2005/05/20 02:47:56;  author: gilles;  state: Exp;  lines: +5 -37
+Removed unused code
+----------------------------
+revision 1.130
+date: 2005/05/20 02:40:26;  author: gilles;  state: Exp;  lines: +23 -15
+Added --useheader option
+----------------------------
 revision 1.129
 date: 2005/05/19 01:54:31;  author: gilles;  state: Exp;  lines: +14 -11
 Added --buffersize option.

README

@@ -2,7 +2,7 @@ NAME
     imapsync - IMAP synchronization, copy or migration tool. Synchronize
     mailboxes between two imap servers. Good at IMAP migration.
 
-    $Revision: 1.129 $
+    $Revision: 1.132 $
 
 INSTALL
      imapsync works fine under any Unix OS.
@@ -29,25 +29,31 @@ SYNOPSIS
       imapsync
 
       imapsync [--host1 server1]  [--port1 <num>]
-               [--user1 <string>] [--passfile1 <string>] 
+               [--user1 <string>] [--passfile1 <string>]
                [--host2 server2]  [--port2 <num>]
-               [--user2 <string>] [--passfile2 <string>] 
+               [--user2 <string>] [--passfile2 <string>]
+               [--noauthmd5]
                [--folder <string> --folder <string> ...]
                [--include <regex>] [--exclude <regex>]
-               [--prefix2 <string>]
+               [--prefix2 <string>] [--regextrans2 <regex>]
                [--sep1 <char>]
                [--sep2 <char>]
+               [--justfolders] [--justconnect]
                [--syncinternaldates]
+               [--buffersize  <int>]
+               [--syncacls]
+               [--regexmess <regex>] [--regexmess <regex>] 
                [--maxsize <int>]
                [--maxage <int>]
                [--skipheader <regex>]
+               [--useheader <string>] [--useheader <string>]
                [--skipsize]
                [--delete] [--expunge]
                [--subscribed] [--subscribe]
-               [--foldersizes]
+               [--nofoldersizes]
                [--dry]
                [--debug] [--debugimap]
-               [--timeout <int>]
+               [--timeout <int>] [--fast]
                [--version] [--help]
   
 DESCRIPTION
@@ -247,5 +253,5 @@ AUTHOR
     teaching free open and gratis softwares. Don't hesitate to pay him for
     that services.
 
-    $Id: imapsync,v 1.129 2005/05/19 01:54:31 gilles Exp $
+    $Id: imapsync,v 1.132 2005/05/20 03:18:48 gilles Exp $
 

TODO

@@ -1,9 +1,10 @@
 TODO file for imapsync
 ----------------------
 
-Look at http://barnson.org/node/81
 
-Update doc with ALL options.
+Interface with external software like procmail 
+
+Look at http://barnson.org/node/81
 
 Add in doc:
 > The problem seems to be that one of the imap servers (the destination
@@ -48,7 +49,6 @@ Look at http://freshmeat.net/projects/freshmeat-submit/
 Look at comp.mail.imap http://groups.google.fr/groups?q=imapsync
 
 
-Think about speed.
 
 Explain expunge behavior
 
@@ -56,6 +56,10 @@ Add a --recurse option when --folder option is used.
 
 Add --prefix1 option. Don't know what is the need exactly.
 
+DONE. Update doc with ALL options.
+
+DONE. Think about speed.
+
 DONE. Pb if "to separator" is in "from folder" name.
 Have to choose a caracter != to separator and
 not in from folders. The solution can be to

VERSION

@@ -1 +1 @@
-1.129
+1.132

imapsync

@@ -6,7 +6,7 @@ imapsync - IMAP synchronization, copy or migration
 tool. Synchronize mailboxes between two imap servers. Good
 at IMAP migration.
 
-$Revision: 1.129 $
+$Revision: 1.132 $
 
 =head1 INSTALL
 
@@ -35,25 +35,31 @@ $Revision: 1.129 $
   imapsync
 
   imapsync [--host1 server1]  [--port1 <num>]
-           [--user1 <string>] [--passfile1 <string>] 
+           [--user1 <string>] [--passfile1 <string>]
            [--host2 server2]  [--port2 <num>]
-           [--user2 <string>] [--passfile2 <string>] 
+           [--user2 <string>] [--passfile2 <string>]
+           [--noauthmd5]
            [--folder <string> --folder <string> ...]
-	   [--include <regex>] [--exclude <regex>]
+           [--include <regex>] [--exclude <regex>]
-           [--prefix2 <string>]
+           [--prefix2 <string>] [--regextrans2 <regex>]
            [--sep1 <char>]
            [--sep2 <char>]
+           [--justfolders] [--justconnect]
            [--syncinternaldates]
-	   [--maxsize <int>]
+           [--buffersize  <int>]
-	   [--maxage <int>]
+           [--syncacls]
+           [--regexmess <regex>] [--regexmess <regex>] 
+           [--maxsize <int>]
+           [--maxage <int>]
            [--skipheader <regex>]
+           [--useheader <string>] [--useheader <string>]
            [--skipsize]
            [--delete] [--expunge]
            [--subscribed] [--subscribe]
-           [--foldersizes]
+           [--nofoldersizes]
            [--dry]
            [--debug] [--debugimap]
-           [--timeout <int>]
+           [--timeout <int>] [--fast]
            [--version] [--help]
   
 =cut
@@ -289,7 +295,7 @@ Gilles LAMIRAL earn his living writing, installing,
 configuring and teaching free open and gratis
 softwares. Don't hesitate to pay him for that services.
 
-$Id: imapsync,v 1.129 2005/05/19 01:54:31 gilles Exp $
+$Id: imapsync,v 1.132 2005/05/20 03:18:48 gilles Exp $
 
 =cut
 
@@ -313,7 +319,8 @@ my(
         $sep1, $sep2,
 	$syncinternaldates, $syncacls,
 	$maxsize, $maxage,
-        $skipheader, $skipsize, $foldersizes, $buffersize,
+        $skipheader, @useheader,
+        $skipsize, $foldersizes, $buffersize,
 	$delete, $expunge, $dry, 
         $authmd5,
         $subscribed, $subscribe,
@@ -333,7 +340,7 @@ my(
 use vars qw ($opt_G); # missing code for this will be option.
 
 
-$rcs = ' $Id: imapsync,v 1.129 2005/05/19 01:54:31 gilles Exp $ ';
+$rcs = ' $Id: imapsync,v 1.132 2005/05/20 03:18:48 gilles Exp $ ';
 $rcs =~ m/,v (\d+\.\d+)/;
 $VERSION = ($1) ? $1 : "UNKNOWN";
 
@@ -370,8 +377,8 @@ $error=0;
 
 my $banner = join("", 
 		  '$RCSfile: imapsync,v $ ',
-		  '$Revision: 1.129 $ ',
+		  '$Revision: 1.132 $ ',
-		  '$Date: 2005/05/19 01:54:31 $ ',
+		  '$Date: 2005/05/20 03:18:48 $ ',
 		  "\n",
 		  "Mail::IMAPClient version used here is ",
 		  $VERSION_IMAPClient, " auth md5 : $md5_supported",
@@ -404,6 +411,8 @@ $authmd5 = (defined($authmd5)) ? $authmd5 : 1;
 $syncacls = (defined($syncacls)) ? $syncacls : 0;
 $foldersizes = (defined($foldersizes)) ? $foldersizes : 1;
 
+@useheader = ("ALL") unless (@useheader);
+
 print "From imap server [$host1] port [$port1] user [$user1]\n";
 print "To   imap server [$host2] port [$port2] user [$user2]\n";
 
@@ -444,6 +453,11 @@ $from = login_imap($host1, $port1, $user1, $password1, $debugimap, $timeout);
 $debugimap and print "To  connection\n";
 $to = login_imap($host2, $port2, $user2, $password2, $debugimap, $timeout);
 
+# No history
+$from->Clear(1);
+$to->Clear(1);
+
+
 sub login_imap {
 	my($host, $port, $user, $password, 
 	   $debugimap, $timeout, $authmech) = @_;
@@ -745,21 +759,16 @@ FOLDER: foreach my $f_fold (@f_folders) {
 	my %f_hash = ();
 	my %t_hash = ();
 	
-	# No history
-	$from->Clear(1);
-	$to->Clear(1);
 
 	
 	print "From Buffer I/O : ", $from->Buffer(), "\n";
 	print "To   Buffer I/O : ", $to->Buffer(), "\n";
-#	$from->Buffer(4096*64);
-#	$to->Buffer(4096*64);
-#	print "From Buffer I/O : ", $from->Buffer(), "\n";
-#	print "To   Buffer I/O : ", $to->Buffer(), "\n";
 
+	
 	print "++++ From Parse 1 ++++\n";
 	
-	my $f_heads = $from->parse_headers([@f_msgs],"ALL") if (@f_msgs) ;
+	my $f_heads = $from->parse_headers($from->Range([@f_msgs]),@useheader) 
+	  if (@f_msgs) ;
 	print "Time headers: ", timenext(), " s\n";
 	my $f_size  = $from->fetch_hash("RFC822.SIZE") if (@f_msgs);
 	print "Time sizes  : ", timenext(), " s\n";
@@ -776,7 +785,8 @@ FOLDER: foreach my $f_fold (@f_folders) {
 	print "Time headers: ", timenext(), " s\n";
 	
 	print "\n++++ To   Parse 1 ++++\n";
-	my $t_heads =   $to->parse_headers([@t_msgs],"ALL") if (@t_msgs);
+	my $t_heads =   $to->parse_headers($to->Range([@t_msgs]),@useheader) 
+	  if (@t_msgs);
 	print "Time headers: ", timenext(), " s\n";
 	my $t_size  =   $to->fetch_hash("RFC822.SIZE") if (@t_msgs);
 	print "Time sizes  : ", timenext(), " s\n";
@@ -993,6 +1003,7 @@ sub get_options
                                    "help"        => \$help,
                                    "timeout=i"   => \$timeout,
 				   "skipheader=s" => \$skipheader,
+				   "useheader=s" => \@useheader,
 				   "skipsize!"   => \$skipsize,
                                   );
 
@@ -1030,7 +1041,7 @@ sub parse_header_msg1 {
 			$debug and print "${s}H $h:", $val, "\n";
 			if ($skipheader and $h =~ m/$skipheader/) {
 				$debug and print "Skipping header $h\n";
-		      next;
+				next;
 			}
 			$headstr .= "$h:". $val;
 		}
@@ -1047,38 +1058,6 @@ sub parse_header_msg1 {
 	$s_hash->{"$m_md5:$size"}{'m'} = $m_uid;
 }
 
-sub parse_header_msg {
-
-	my ($m, $imap, $s, $s_hash) = @_;
-	$debug and print "-" x 50, "\nMSG $m\n";
-	my $head = $imap->parse_headers($m,"ALL");
-	my $headstr;
-	$debug and print "Head NUM:", scalar(keys(%$head)), "\n";
-	# no header -> return
-	return unless(scalar(keys(%$head)));	    
-	foreach my $h (sort keys(%$head)){
-		foreach my $val (sort @{$head->{$h}}) {
-			# no 8-bit data in headers !
-			$val =~ s/[\x80-\xff]/X/g;
-			# remove the first blanks (dbmail bug ?)
-			$val =~ s/^\s+//;
-			# show stuff in debug mode
-			$debug and print "${s}H $h:", $val, "\n";
-			if ($skipheader and $h =~ m/$skipheader/) {
-				$debug and print "Skipping header $h\n";
-				next;
-			}
-			$headstr .= "$h:". $val;
-		}
-	}
-	my $m_md5 = md5_base64($headstr);
-	my $size = $imap->size($m);
-	$debug and print "$s msg $m:$m_md5:$size\n";
-	$size = 0 if ($skipsize);
-	$s_hash->{"$m_md5:$size"}{'5'} = "$m_md5:$size";
-	$s_hash->{"$m_md5:$size"}{'s'} = $size;
-	$s_hash->{"$m_md5:$size"}{'m'} = $m;
-}
 
 sub  firstline {
         # extract the first line of a file (without \n)
@@ -1110,7 +1089,8 @@ Several options are mandatory.
 --user2       <string> : user to login.   Mandatory.
 --password2   <string> : password for the user2. Dangerous, use --passfile2
 --passfile2   <string> : password file for the user2. Contains the password.
---noauthmd5            : don't use MD5 authentification
+--noauthmd5            : don't use MD5 authentification.
+--authmd5              : use MD5 authentification.
 --folder      <string> : sync only this folder.
 --folder      <string> : and this one.
 --folder      <string> : and this one, etc.
@@ -1148,6 +1128,9 @@ Several options are mandatory.
                          final stats (skipped) don't count older messages
 --skipheader  <regex>  : Don't take into account header keyword 
                          matching <string> ex: --skipheader 'X.*'
+--useheader   <string> : Use this header to compare messages on both sides.
+                         Ex: Message-ID or Subject or Date.
+--useheader   <string>   and this one, etc.
 --skipsize             : Don't take message size into account.
 --dry                  : do nothing, just print what would be done.
 --subscribed           : transfer only subscribed folders.
@@ -1162,7 +1145,7 @@ Several options are mandatory.
 --debug                : debug mode.
 --debugimap            : imap debug mode.
 --version              : print sotfware version.
---justconnect          : just connect to both servers and print useful 
+--justconnect          : just connect to both servers and print useful
                          information.
 --justfolders          : just do things about folders (ignore messages).
 --fast                 : be faster.

tests.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# $Id: tests.sh,v 1.37 2005/05/19 01:50:02 gilles Exp $	
+# $Id: tests.sh,v 1.38 2005/05/20 02:51:33 gilles Exp $	
 
 #### Shell pragmas
 
@@ -476,7 +476,8 @@ big_transfert()
 	--host2 plume --user2 tete@est.belle \
 	--passfile2 /var/tmp/secret.tete \
 	--subscribed --foldersizes --noauthmd5 \
-        --fast --folder INBOX.Backup || \
+        --fast --folder INBOX.Backup \
+	--useheader Message-ID --useheader Received || \
     true
     }
     date2=`date`
@@ -576,6 +577,27 @@ essnet_plume2()
         --prefix2 INBOX. --regextrans2 's¤INBOX.INBOX¤INBOX¤'
 }
 
+useheader() 
+{
+	if test X`hostname` = X"plume"; then
+		echo3 Here is plume
+		
+		./imapsync \
+		--host2 plume --user2 tata@est.belle \
+		--passfile2 /var/tmp/secret.tata \
+		--host1 loul  --user1 tata \
+		--passfile1 /var/tmp/secret.tata \
+		--folder INBOX.yop.yap \
+		--useheader 'Message-ID' \
+		--dry --debug
+		
+		echo 'rm /home/vmail/tata/.yop.yap/cur/*'
+	else
+		:
+	fi
+}
+
+
 regexmess() 
 {
 	if test X`hostname` = X"plume"; then
@@ -633,7 +655,8 @@ test $# -eq 0 && run_tests \
 	foldersizes \
 	big_transfert_sizes_only \
 	regexmess \
-	
+	useheader \
+
 
 
 # selective tests