diff --git a/ChangeLog b/ChangeLog index 19aff35..42234dd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,17 +1,35 @@ RCS file: RCS/imapsync,v Working file: imapsync -head: 1.333 +head: 1.337 branch: locks: strict - gilles: 1.333 + gilles: 1.337 access list: symbolic names: keyword substitution: kv -total revisions: 333; selected revisions: 333 +total revisions: 337; selected revisions: 337 description: ---------------------------- -revision 1.333 locked by: gilles; +revision 1.337 locked by: gilles; +date: 2010/07/16 23:23:40; author: gilles; state: Exp; lines: +12 -6 +Added --usedatemanip option (not documented in --help) +Turned off Date::Manip usage by default: release 6.x vs 5.x buggy, +and dprof showed slow routines come from Date::Manip +---------------------------- +revision 1.336 +date: 2010/07/16 22:04:25; author: gilles; state: Exp; lines: +8 -7 +Bugfix. $syncinternaldates could not be set 0. Unbelievable! +---------------------------- +revision 1.335 +date: 2010/07/16 00:44:25; author: gilles; state: Exp; lines: +26 -15 +Bugfix. Fixed Date_TimeZone break under win32. +---------------------------- +revision 1.334 +date: 2010/07/15 12:20:48; author: gilles; state: Exp; lines: +50 -32 +Added statistics about "Messages void (noheader) on hostX" +---------------------------- +revision 1.333 date: 2010/07/14 23:55:13; author: gilles; state: Exp; lines: +7 -9 docfix ---------------------------- diff --git a/FAQ b/FAQ index e78cc41..5818f77 100644 --- a/FAQ +++ b/FAQ @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: FAQ,v 1.70 2010/07/12 00:18:33 gilles Exp gilles $ +# $Id: FAQ,v 1.71 2010/07/15 12:07:00 gilles Exp gilles $ +------------------+ | FAQ for imapsync | @@ -91,7 +91,10 @@ R. - Download latest Mail::IMAPClient 3.xx at - run imapsync with perl and -I option tailing to use the perl module Mail-IMAPClient-3.xx. Example: - perl -I./Mail-IMAPClient-3.23/lib imapsync ... + perl -I./Mail-IMAPClient-3.23/lib ./imapsync ... + + or if imapsync is in directory /path/ + perl -I./Mail-IMAPClient-3.23/lib /path/imapsync ... ======================================================================= Q. imapsync does not work with Mail::IMAPClient 3.xx @@ -102,10 +105,12 @@ R. - Download Mail::IMAPClient 2.2.9 at http://search.cpan.org/CPAN/authors/id/D/DJ/DJKERNEN/Mail-IMAPClient-2.2.9.tar.gz - untar it anywhere: tar xzvf Mail-IMAPClient-2.2.9.tar.gz + - run imapsync with perl and -I option tailing to use Mail-IMAPClient-2.2.9: - perl -I./Mail-IMAPClient-2.2.9 imapsync [...] + perl -I./Mail-IMAPClient-2.2.9 ./imapsync [...] + or if imapsync is in directory /path/ - perl -I./Mail-IMAPClient-2.2.9 /path/imapsync [...] + perl -I./Mail-IMAPClient-2.2.9 /path/imapsync [...] ======================================================================= Q. Can I use imapsync to migrate emails from pop server to imap server? @@ -162,6 +167,23 @@ c) Don't use buggy Eudora. d) Use the --syncinternaldates option and keep using Eudora. --syncinternaldates is now turn on by default. + +======================================================================= +Q. imapsync calculates 479 messages in a folder but only transfers 400 + messages. What's happen? + +R. imapsync considers the header part of a message (as a whole or + only specific lines depending on --useheader --skipheader) + to identify a message on both sides. + +Two consequences: + + 1) Messages with no header are not transferred. + 2) Duplicate messages (identical header) are not transferred + several times. + +The result is that you can have more messages on host1 than on host2. + ======================================================================= Q. Couldn't create [INBOX.Ops/foo/bar]: NO Invalid mailbox name: INBOX.Ops/foo/bar @@ -208,6 +230,8 @@ only the flags listed in PERMANENTFLAGS. Some imap servers have problems with flags not beginning with the backslash character \ +(see next question to find a solution to this issue) + ======================================================================= Q. I need to keep only a defind list of flags, how can I do? @@ -245,6 +269,9 @@ Any other flag must begin with another character. System flags are just flags defined by an RFC instead of by users. Conclusion, some imap server coders don't read the RFCs (so do I). +recent imapsync deals with this issue by filter with PERMANENTFLAGS +automatically. + ======================================================================= Q. Flags are not well synchonized. Is it a bug? @@ -296,6 +323,8 @@ timed out on the destination server. Try options : --nofoldersizes --useheader Message-ID --fast +recent imapsync reconnects automatically. + ======================================================================= Q. imapsync failed with a "word too long" error from the imap server, What can I do? @@ -430,20 +459,23 @@ From folders list : [INBOX] [public.dreams] [etc.] ====================================================================== Q. I want the --folder 'MyFolder' option be recursive. +Two solutions: + +R. Use + +--folderrec 'MyFolder' + R. Do not use the --folder option. Instead, use --include '^MyFolder' Then the folder "MyFolder" and all its subfolders will be handled and only them. -R. Use - ---folderrec 'MyFolder' - ====================================================================== Q. How to migrate from uw-imap with an admin/authuser account? R. Use + --user1="user*admin_user" --password1 "admin_user_password" @@ -479,7 +511,7 @@ Here is an example: Q. Is there anyway of making imapsync purge the destination folder when the source folder is deleted? -R. No, that's too dangerous. +R. No, that's too dangerous. May be coded in future release. But if the source folder is empty (not deleted) and options --delete2 --expunge2 are used then the destination folder will be empty. @@ -489,6 +521,10 @@ Q. Is it possible to synchronize all messages from one server to another without recreating the folder structure and the target server. R. Yes. + +For example, to synchronize all messages in all forders on host1 +to folder INBOX only on host2: + 1) First try (safe mode): imapsync \ @@ -530,7 +566,9 @@ imapsync \ ======================================================================= Q. Give examples about --regextrans2 -R. Examples: +R. --regextrans2 is used to transform folder names + +Examples: 0) First try with --dry --justfolders options since imapsync shows the transformations it will do without really doing them. Then when @@ -603,7 +641,7 @@ Q. I want to play with headers line and --regexmess but I want to leave R. The header/body separation is a blank line so an example: --regexmess 's{\A(.*?(?! ^$))^Date:(.*?)$}{$1Date:$2\nX-Date:$2}gxms' -Will replace +Will replace (HeaderBegin and HeaderEnd are not part of the header) HeaderBegin Message-ID: <499EF800.4030002@blabla.fr> @@ -639,23 +677,24 @@ Received: ... Any Maildir/ configured imap server may refuse this message since its header is invalid. The first "From " line is not valid. It lacks a -colon character ":". To solve this problem you have several solutions +colon character ":". To solve this issue you have several solutions a) Remove manually this first "From " line for each message before using imapsync. b) Replace manually the whitespace by a colon in string "From " but you - might end with two "From:" lines (just have a look at the other header lines) + might end with two "From:" lines (just have a look at the other + header lines of the message) -c) Run imapsync with the following option : +c) Run imapsync with the following option (this replaces "From "by "From:"): --regexmess 's/\AFrom /From:/' or may be better (no other "From:" collision): -d) Run imapsync with the following option : +d) Run imapsync with the following option (this replaces "From "by "X-om:"): --regexmess 's/\AFrom /X-om:/' -e) Run imapsync with the following option : +e) Run imapsync with the following option (this removes the whole "From " line): --regexmess 's{\AFrom\ [^\n]*(\n)?}{}gxms' Solution e) is solution a) made by imapsync itself. @@ -679,6 +718,10 @@ using the same server, we can use $from->copy Therefore we seem to not download and upload the message and therefore we do not have any format issues. And now it works fine. (Thanks to Hansjoerg.Maurer) +======================================================================= + Server specific issues and solutions +======================================================================= + ======================================================================= Q. Synchronising from XXX to Gmail @@ -687,8 +730,7 @@ R. There are some details to get the special [Gmail] sub-folders right. Here's an example of migrating an old "Sent" folder to Gmail's structure: -imapsync --syncinternaldates \ - --host1 mail.oldhost.com \ +imapsync --host1 mail.oldhost.com \ --user1 my_email@oldhost.com \ --password1 password \ --host2 imap.gmail.com --port2 993 --ssl2 \ @@ -761,9 +803,12 @@ Q. I'm migrating from WU to Cyrus, and the mail folders are under /home/user/mail but the tool copies everything in /home/user, how can i avoid that? +Two solutions: + R. Use imapsync ... --include '^mail' -or (better) + +R. or (better) imapsync ... --subscribed --subscribe diff --git a/INSTALL b/INSTALL index c772481..412fca8 100644 --- a/INSTALL +++ b/INSTALL @@ -1,4 +1,4 @@ -# $Id: INSTALL,v 1.16 2010/01/14 23:41:59 gilles Exp gilles $ +# $Id: INSTALL,v 1.17 2010/07/16 22:01:57 gilles Exp gilles $ # # INSTALL file for imapsync # imapsync : IMAP sync or copy tool. @@ -58,15 +58,20 @@ Here is some individual module help: http://search.cpan.org/~djkernen/ http://search.cpan.org/~djkernen/Mail-IMAPClient-2.2.9/ - In fact I use Mail-IMAPClient-2.2.9 + In fact I use Mail-IMAPClient-2.2.9 and latest Mail-IMAPClient-2.xx + (xx >= 25) To know the version you have on your system try : perl -mMail::IMAPClient -e 'print $Mail::IMAPClient::VERSION, "\n"' New Mail-IMAPClient-3.xx works now with imapsync, - at least with Mail-IMAPClient-3.19 (previous may bug) + at least with Mail-IMAPClient-3.25 (previous may bug) - Perl Digest::MD5 module. + try: + + perl -mDigest::MD5 + http://search.cpan.org/ http://search.cpan.org/~gaas/Digest-MD5-2.36/ To know the version you have on your system try : @@ -88,9 +93,12 @@ Here is some individual module help: - File::Path perl -mFile::Path -e '' -- (Option) Perl Digest::HMAC_MD5 module +- Perl Digest::HMAC_MD5 module Good for non plain text password over network. +Everything in one command: + + perl -mMail::IMAPClient -mDigest::MD5 -mTerm::ReadKey -mIO::Socket::SSL -mDate::Manip -mFile::Spec -mDigest::HMAC_MD5 -e '' INSTALLING ---------- diff --git a/Makefile b/Makefile index f19a42e..9cd1707 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -# $Id: Makefile,v 1.30 2010/07/12 12:14:11 gilles Exp gilles $ +# $Id: Makefile,v 1.32 2010/07/17 00:17:18 gilles Exp gilles $ TARGET=imapsync @@ -103,8 +103,7 @@ tarball: echo making tarball $(DIST_FILE) mkdir -p dist mkdir -p ../prepa_dist/$(DIST_NAME) - rsync -aCv --delete --omit-dir-times --exclude dist/ ./ ../prepa_dist/$(DIST_NAME) - #sync && sync && sync && sleep 2 + rsync -aCv --delete --omit-dir-times --exclude dist/ ./ ../prepa_dist/$(DIST_NAME) cd ../prepa_dist && (tar czfv $(DIST_FILE) $(DIST_NAME) || tar czfv $(DIST_FILE) $(DIST_NAME)) ln -f ../prepa_dist/$(DIST_FILE) dist/ cd dist && md5sum $(DIST_FILE) > $(DIST_FILE).md5.txt diff --git a/README b/README index 175fd87..ff85705 100644 --- a/README +++ b/README @@ -3,7 +3,7 @@ NAME Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 32 different IMAP server softwares supported with success. - $Revision: 1.333 $ + $Revision: 1.337 $ INSTALL imapsync works fine under any Unix OS with perl. @@ -24,8 +24,8 @@ INSTALL Go into the directory imapsync-x.xx and read the INSTALL file. The INSTALL file is also at http://www.linux-france.org/prj/imapsync/INSTALL (for windows users) - - The freshmeat record is at http://freshmeat.net/projects/imapsync/ + + The freshmeat record is at http://freshmeat.net/projects/imapsync/ SYNOPSIS imapsync [options] @@ -76,7 +76,7 @@ SYNOPSIS [--pidfile ] [--tmpdir ] [--version] [--help] - + DESCRIPTION The command imapsync is a tool allowing incremental and recursive imap transfer from one mailbox to another. @@ -204,13 +204,10 @@ AUTHOR that services. BUG REPORT GUIDELINES - Report any bugs or feature requests to the public mailing-list or to the - author. - Help us to help you: follow the following guidelines. - One time in your life, read the paper "How To Ask Questions The Smart - Way" http://www.catb.org/~esr/faqs/smart-questions.html + Report any bugs or feature requests to the public mailing-list or to the + author. Before reporting bugs, read the FAQ, the README and the TODO files. http://www.linux-france.org/prj/imapsync/ @@ -218,7 +215,8 @@ BUG REPORT GUIDELINES Upgrade to last imapsync release, maybe the bug is already fixed. Upgrade to last Mail-IMAPClient Perl module. - http://search.cpan.org/dist/Mail-IMAPClient/ + http://search.cpan.org/dist/Mail-IMAPClient/ maybe the bug is already + fixed. Make a good title with word "imapsync" in it (my spam filter won't filter it), Don't write an email title with just "imapsync" or @@ -231,18 +229,30 @@ BUG REPORT GUIDELINES Help us to help you: in your report, please include: - imapsync version. - - output given with --debug --debugimap near the failure point. - Isolate a message in a folder 'BUG' and use --folder 'BUG' - - imap server software on both side and their version number. - - imapsync with all the options you use, the full command line + + - output given with --debug --debugimap near the failure point. + Isolate a message or two in a folder 'BUG' and use + --folder 'BUG' --debug --debugimap + + - imap server software on both side and their version number. + + - imapsync with all the options you use, the full command line you use (except the passwords of course). - - IMAPClient.pm version. - - operating system running imapsync. - - operating systems on both sides and the third side in case + + - IMAPClient.pm version. + + - operating system running imapsync. + + - operating systems on both sides and the third side in case you run imapsync on a foreign host from the both. - - virtual software context (vmware, xen etc.) + + - virtual software context (vmware, xen etc.) - Most of those values can be found as a copy/paste at the begining of the output. + Most of those values can be found as a copy/paste at the begining of the + output. + + One time in your life, read the paper "How To Ask Questions The Smart + Way" http://www.catb.org/~esr/faqs/smart-questions.html and forget it. IMAP SERVERS Failure stories reported with the following 4 imap servers: @@ -377,5 +387,5 @@ SIMILAR SOFTWARES Feedback (good or bad) will always be welcome. - $Id: imapsync,v 1.333 2010/07/14 23:55:13 gilles Exp gilles $ + $Id: imapsync,v 1.337 2010/07/16 23:23:40 gilles Exp gilles $ diff --git a/TIME b/TIME index 3de0531..5a78c86 100644 --- a/TIME +++ b/TIME @@ -1,4 +1,8 @@ +140 imapsync under win32 strawberry, pp (par::packer), imapsync.exe +300 imapsync under win32. test msw, environnement, using activestate +70 Profiling tests. Interresting results! +110 FAQ review, FAQ item "messages not all transferred", imapsync stats about "noheader" 250 Reviewed varaibles names, statistics, sub definitions from top to connect_imap() 120 W3C validation of web site 120 --pidfile option. die_clean() exit_clean() diff --git a/VERSION b/VERSION index 9de786a..02ce9f6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.333 +1.337 diff --git a/imapsync b/imapsync index 34a4c00..2cb6d44 100755 --- a/imapsync +++ b/imapsync @@ -19,7 +19,7 @@ tool. Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 32 different IMAP server softwares supported with success. -$Revision: 1.333 $ +$Revision: 1.337 $ =head1 INSTALL @@ -247,14 +247,10 @@ softwares. Do not hesitate to pay him for that services. =head1 BUG REPORT GUIDELINES -Report any bugs or feature requests to the public mailing-list -or to the author. - Help us to help you: follow the following guidelines. -One time in your life, read the paper -"How To Ask Questions The Smart Way" -http://www.catb.org/~esr/faqs/smart-questions.html +Report any bugs or feature requests to the public mailing-list +or to the author. Before reporting bugs, read the FAQ, the README and the TODO files. http://www.linux-france.org/prj/imapsync/ @@ -264,10 +260,11 @@ is already fixed. Upgrade to last Mail-IMAPClient Perl module. http://search.cpan.org/dist/Mail-IMAPClient/ +maybe the bug is already fixed. Make a good title with word "imapsync" in it (my spam filter won't filter it), Don't write an email title with just "imapsync" or "problem", -a good title is made of keywords summary, not too long (one visible line). +a good title is made of keywords summary, not too long (one visible line). Don't write imapsync in uppercase in the email title, we'll know you run windows(tm) and you haven't read the README yet. @@ -275,18 +272,31 @@ know you run windows(tm) and you haven't read the README yet. Help us to help you: in your report, please include: - imapsync version. + - output given with --debug --debugimap near the failure point. - Isolate a message in a folder 'BUG' and use --folder 'BUG' + Isolate a message or two in a folder 'BUG' and use + --folder 'BUG' --debug --debugimap + - imap server software on both side and their version number. + - imapsync with all the options you use, the full command line you use (except the passwords of course). + - IMAPClient.pm version. + - operating system running imapsync. + - operating systems on both sides and the third side in case you run imapsync on a foreign host from the both. + - virtual software context (vmware, xen etc.) - Most of those values can be found as a copy/paste at the begining of the output. +Most of those values can be found as a copy/paste at the begining of the output. + +One time in your life, read the paper +"How To Ask Questions The Smart Way" +http://www.catb.org/~esr/faqs/smart-questions.html +and forget it. =head1 IMAP SERVERS @@ -444,7 +454,7 @@ Entries for imapsync: Feedback (good or bad) will always be welcome. -$Id: imapsync,v 1.333 2010/07/14 23:55:13 gilles Exp gilles $ +$Id: imapsync,v 1.337 2010/07/16 23:23:40 gilles Exp gilles $ =cut @@ -486,6 +496,7 @@ my( $sep1, $sep2, $syncinternaldates, $idatefromheader, + $usedatemanip, $syncacls, $fastio1, $fastio2, $maxsize, $maxage, $minage, @@ -507,6 +518,8 @@ my( $nb_msg_skipped_dry_mode, $h1_nb_msg_duplicate, $h2_nb_msg_duplicate, + $h1_nb_msg_noheader, + $h2_nb_msg_noheader, $h1_total_bytes_duplicate, $h2_total_bytes_duplicate, $h1_nb_msg_deleted, @@ -529,7 +542,7 @@ my( # global variables initialisation -$rcs = '$Id: imapsync,v 1.333 2010/07/14 23:55:13 gilles Exp gilles $ '; +$rcs = '$Id: imapsync,v 1.337 2010/07/16 23:23:40 gilles Exp gilles $ '; $total_bytes_transferred = 0; $total_bytes_skipped = 0; @@ -537,7 +550,8 @@ $total_bytes_error = 0; $nb_msg_transferred = 0; $nb_msg_skipped = $nb_msg_skipped_dry_mode = 0; $h1_nb_msg_deleted = $h2_nb_msg_deleted = 0; -$h1_nb_msg_duplicate = $h2_nb_msg_duplicate =0; +$h1_nb_msg_duplicate = $h2_nb_msg_duplicate = 0; +$h1_nb_msg_noheader = $h2_nb_msg_noheader = 0; $h1_total_bytes_duplicate = $h2_total_bytes_duplicate = 0; $nb_errors = 0; @@ -559,8 +573,8 @@ $pidfile ||= $tmpdir . '/imapsync.pid'; # allow Mail::IMAPClient 3.0.xx by default $allow3xx = defined($allow3xx) ? $allow3xx : 1; -check_lib_version() or - die "imapsync needs perl lib Mail::IMAPClient release 2.2.9, or 3.0.25 or superior \n"; +# Does not use Date::Manip by default: buggy 5.x vs 6.x and slow +$usedatemanip = defined($usedatemanip) ? $usedatemanip : 0; print banner_imapsync(@argv_copy); @@ -570,6 +584,8 @@ is_valid_directory($tmpdir); write_pidfile($pidfile) if ($pidfile); print "Modules version list:\n", modules_VERSION(), "\n"; +check_lib_version() or + die "imapsync needs perl lib Mail::IMAPClient release 2.2.9, or 3.0.25 or superior \n"; exit_clean(0) if ($justbanner); @@ -635,7 +651,7 @@ if ($justconnect) { $user1 || missing_option("--user1"); $user2 || missing_option("--user2"); -$syncinternaldates = defined($syncinternaldates) ? defined($syncinternaldates) : 1; +$syncinternaldates = defined($syncinternaldates) ? $syncinternaldates : 1; if($idatefromheader) { print "Turned ON idatefromheader, ", @@ -651,16 +667,28 @@ if ($syncinternaldates) { } if ($syncinternaldates || $idatefromheader) { + + # Date::Manip is an ugly module it exits (confess) for reading an unset value + # I should write a bug report but too lazy. + no warnings 'redefine'; local *Carp::confess = sub { return undef; }; require Date::Manip; Date::Manip->import(qw(ParseDate UnixDate Date_Init Date_TimeZone)); - #print "Date_init: [", join(" ",Date_Init()), "]\n"; - print "TimeZone:[", Date_TimeZone(), "]\n"; - if (not (Date_TimeZone())) { - warn "TimeZone not defined, setting it to GMT"; - Date_Init("TZ=GMT"); + + if ($OSNAME eq "MSWin32") { + # It seems that local *Carp does not work on win32 + my $TZ = $ENV{TZ} || 'GMT'; + Date_Init("TZ=$TZ"); print "TimeZone: [", Date_TimeZone(), "]\n"; + }else{ + #print "Date_init: [", join(" ",Date_Init()), "]\n"; + print "TimeZone:[", Date_TimeZone(), "]\n"; + if (not (Date_TimeZone())) { + warn "TimeZone not defined, setting it to GMT"; + Date_Init("TZ=GMT"); + print "TimeZone: [", Date_TimeZone(), "]\n"; + } } } @@ -1196,7 +1224,7 @@ foreach my $h1_fold (@h1_folders) { if ($foldersizes) { foldersizes("Host1", $imap1, \@h1_folders); - foldersizes("Host2", $imap2, \@h2_folders); + foldersizes("Host2", $imap2, \@h2_folders); } @@ -1610,11 +1638,13 @@ FOLDER: foreach my $h1_fold (@h1_folders) { print "+ Skipping msg #$m:$h1_size on host1 folder $h1_fold (no header so we ignore this message)\n"; $total_bytes_skipped += $h1_size; $nb_msg_skipped += 1; + $h1_nb_msg_noheader +=1; } elsif(0 == $rc) { # duplicate push(@h1_msgs_duplicate, $m); # duplicate, same id same size? my $h1_size = $h1_fir->{$m}->{"RFC822.SIZE"} || 0; + $nb_msg_skipped += 1; $h1_total_bytes_duplicate += $h1_size; $h1_nb_msg_duplicate += 1; } @@ -1639,6 +1669,7 @@ FOLDER: foreach my $h1_fold (@h1_folders) { if (! defined($rc)) { my $h2_size = $h2_fir->{$m}->{"RFC822.SIZE"} || 0; print "+ Skipping msg #$m:$h2_size in host2 folder $h2_fold (no header so we ignore this message)\n"; + $h2_nb_msg_noheader += 1 ; } elsif(0 == $rc) { # duplicate my $h2_size = $h2_fir->{$m}->{"RFC822.SIZE"} || 0; @@ -1847,6 +1878,7 @@ Bye.' sub good_date { my ($d) = @_; + return($d) if (! $usedatemanip); $d = UnixDate(ParseDate($d), "%d-%b-%Y %H:%M:%S %z"); $d = "\"$d\""; return($d); @@ -1863,6 +1895,7 @@ Bye.' print "flags & date from: [$h1_flags][$d]\n"; last FOLDER if $imap1->IsUnconnected(); last FOLDER if $imap2->IsUnconnected(); + $d = undef; unless ($dry) { if ($OSNAME eq "MSWin32") { @@ -2062,7 +2095,7 @@ exit_clean(0); # subroutines sub imapsync_version { - my $rcs = '$Id: imapsync,v 1.333 2010/07/14 23:55:13 gilles Exp gilles $ '; + my $rcs = '$Id: imapsync,v 1.337 2010/07/16 23:23:40 gilles Exp gilles $ '; $rcs =~ m/,v (\d+\.\d+)/; my $VERSION = ($1) ? $1: "UNKNOWN"; return($VERSION); @@ -2146,8 +2179,8 @@ sub banner_imapsync { my @argv_copy = @_; my $banner_imapsync = join("", '$RCSfile: imapsync,v $ ', - '$Revision: 1.333 $ ', - '$Date: 2010/07/14 23:55:13 $ ', + '$Revision: 1.337 $ ', + '$Date: 2010/07/16 23:23:40 $ ', "\n",localhost_info(), "\n", "Command line used:\n", "$0 ", command_line_nopassword(@argv_copy), "\n", @@ -2235,25 +2268,27 @@ sub select_msgs { sub stats { print "++++ Statistics ++++\n"; - print "Transfer time : $timediff sec\n"; - print "Messages transferred : $nb_msg_transferred "; + print "Transfer time : $timediff sec\n"; + print "Messages transferred : $nb_msg_transferred "; print "(could be $nb_msg_skipped_dry_mode without dry mode)" if ($dry); print "\n"; - print "Messages skipped : $nb_msg_skipped\n"; - print "Messages duplicate on host1 : $h1_nb_msg_duplicate\n"; - print "Messages duplicate on host2 : $h2_nb_msg_duplicate\n"; - print "Messages deleted on host1 : $h1_nb_msg_deleted\n"; - print "Messages deleted on host2 : $h2_nb_msg_deleted\n"; - print "Total bytes transferred : $total_bytes_transferred\n"; - print "Total bytes duplicate host1 : $h1_total_bytes_duplicate\n"; - print "Total bytes duplicate host2 : $h2_total_bytes_duplicate\n"; - print "Total bytes skipped : $total_bytes_skipped\n"; - print "Total bytes error : $total_bytes_error\n"; + print "Messages skipped : $nb_msg_skipped\n"; + print "Messages found duplicate on host1 : $h1_nb_msg_duplicate\n"; + print "Messages found duplicate on host2 : $h2_nb_msg_duplicate\n"; + print "Messages void (noheader) on host1 : $h1_nb_msg_noheader\n"; + print "Messages void (noheader) on host2 : $h2_nb_msg_noheader\n"; + print "Messages deleted on host1 : $h1_nb_msg_deleted\n"; + print "Messages deleted on host2 : $h2_nb_msg_deleted\n"; + print "Total bytes transferred : $total_bytes_transferred\n"; + print "Total bytes duplicate host1 : $h1_total_bytes_duplicate\n"; + print "Total bytes duplicate host2 : $h2_total_bytes_duplicate\n"; + print "Total bytes skipped : $total_bytes_skipped\n"; + print "Total bytes error : $total_bytes_error\n"; $timediff ||= 1; # No division per 0 - printf ("Message rate : %.1f messages/s\n", $nb_msg_transferred / $timediff); - printf ("Average bandwidth rate : %.1f KiB/s\n", $total_bytes_transferred / 1024 / $timediff); - print "Reconnections to host1 : $host1_reconnect_count\n"; - print "Reconnections to host2 : $host2_reconnect_count\n"; + printf ("Message rate : %.1f messages/s\n", $nb_msg_transferred / $timediff); + printf ("Average bandwidth rate : %.1f KiB/s\n", $total_bytes_transferred / 1024 / $timediff); + print "Reconnections to host1 : $host1_reconnect_count\n"; + print "Reconnections to host2 : $host2_reconnect_count\n"; print "Detected $nb_errors errors\n\n"; print thank_author(); } @@ -2308,6 +2343,7 @@ sub get_options { "delete2!" => \$delete2, "syncinternaldates!" => \$syncinternaldates, "idatefromheader!" => \$idatefromheader, + "usedatemanip!" => \$usedatemanip, "syncacls!" => \$syncacls, "maxsize=i" => \$maxsize, "maxage=i" => \$maxage, diff --git a/index.shtml b/index.shtml index 24551d3..77ce91f 100644 --- a/index.shtml +++ b/index.shtml @@ -5,7 +5,7 @@ imapsync <!--#exec cmd="cat VERSION" --> - + @@ -82,7 +82,11 @@ Or offer him a book on his

See ChangeLog to know what's new.

-

imapsync download

+

imapsync source download

+ +Standalone imapsync executable for win32, +thanks to Strawberry Perl 5.12 and Par::Packed module.
+(imapsync.exe built time is )

imapsync installation

@@ -138,11 +142,12 @@ If you really want a feature you can donate money and I'll code it.

Some features and their time/money to be done evaluation

- - - - - + + + + + +
Feature Time guessedTime spentMoney receivedMoney still needed
Speedup 50% 10 hours 0 0 $ 300 $
Backup to files 8 hours 60 min 0 $ 240 $
--deletefolder2 3 hours 30 min 0 $ 90 $
NTLM auth 3 hours 60 min 0 $ 90 $
Feature Time guessedTime spentMoney receivedMoney still needed
Speedup 50% 10 hours 80 min 0 $ 300 $
Backup to files 8 hours 60 min 0 $ 240 $
--deletefolder2 3 hours 30 min 0 $ 90 $
NTLM auth 3 hours 60 min 0 $ 90 $
Win32 imapsync.exe 8 hours 520 min 0 $ 240 $
@@ -161,7 +166,7 @@ If you really want a feature you can donate money and I'll code it.
This document last modified
-$Id: index.shtml,v 1.10 2010/07/15 00:06:46 gilles Exp gilles $ +$Id: index.shtml,v 1.14 2010/07/17 00:07:16 gilles Exp gilles $

diff --git a/test.bat b/test.bat new file mode 100755 index 0000000..c40a6ed --- /dev/null +++ b/test.bat @@ -0,0 +1,18 @@ + +%REM $Id: test.bat,v 1.5 2010/07/16 23:27:13 gilles Exp gilles $ + +cd C:\msys\1.0\home\Admin\imapsync +perl -mMail::IMAPClient -mDigest::MD5 -mTerm::ReadKey -mIO::Socket::SSL -mDate::Manip -mFile::Spec -mDigest::HMAC_MD5 -e '' + +set TZ="GMT" +perl ./imapsync --host1 l --user1 toto --passfile1 secret.toto --host2 l --user2 titi --passfile2 secret.titi --noauthmd5 --delete2 --expunge2 +perl ./imapsync --host1 l --user1 tata --passfile1 secret.tata --host2 l --user2 titi --passfile2 secret.titi --noauthmd5 --delete2 --expunge2 --folder INBOX + +% REM -M Date::Manip 6.xx buggy? +pp -o imapsync.exe -M Term::ReadKey -M IO::Socket::SSL -M Digest::HMAC_MD5 imapsync + +.\imapsync.exe --host1 l --user1 toto --passfile1 secret.toto --host2 l --user2 titi --passfile2 secret.titi --noauthmd5 --delete2 --expunge2 + +.\imapsync.exe --host1 l --user1 tata --passfile1 secret.tata --host2 l --user2 titi --passfile2 secret.titi --noauthmd5 --delete2 --expunge2 --folder INBOX + + diff --git a/tests.sh b/tests.sh index b274305..c06746f 100644 --- a/tests.sh +++ b/tests.sh @@ -1,9 +1,9 @@ #!/bin/sh -# $Id: tests.sh,v 1.106 2010/07/14 23:25:14 gilles Exp gilles $ +# $Id: tests.sh,v 1.109 2010/07/16 23:28:12 gilles Exp gilles $ # Example: -# CMD_PERL='perl -I./Mail-IMAPClient-3.14/lib' sh -x tests.sh +# CMD_PERL='perl -I./Mail-IMAPClient-3.25/lib' sh -x tests.sh HOST1=${HOST1:-'localhost'} @@ -11,8 +11,13 @@ echo HOST1=$HOST1 HOST2=${HOST2:-'localhost'} echo HOST2=$HOST2 +# most tests use: CMD_PERL=${CMD_PERL:-'perl -I./Mail-IMAPClient-2.2.9'} +# few debugging tests use: +CMD_PERL_2xx='perl -I./Mail-IMAPClient-2.2.9' +CMD_PERL_3xx='perl -I./Mail-IMAPClient-3.25/lib' + #### Shell pragmas exec 3>&2 # @@ -74,8 +79,6 @@ no_args() { # mailbox tata titi on most ll_*() tests -# tutu@est.belle # not used - # mailbox tete@est.belle # used on big size tests # big_transfert() # big_transfert_sizes_only() @@ -128,8 +131,6 @@ first_sync_dry() { --noauthmd5 --dry } - - first_sync() { $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 toto \ @@ -284,23 +285,34 @@ ll_prefix12() { -ll_internaldate() { - if can_send; then - #echo3 Here is plume - sendtestmessage - else - : - fi - $CMD_PERL ./imapsync \ - --host1 $HOST1 --user1 tata \ - --passfile1 ../../var/pass/secret.tata \ +ll_nosyncinternaldates() { + can_send && sendtestmessage toto + $CMD_PERL_2xx ./imapsync \ + --host1 $HOST1 --user1 toto \ + --passfile1 ../../var/pass/secret.toto \ --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ - --folder INBOX \ - --syncinternaldates \ - --allow3xx -} + --folder INBOX --noauthmd5 \ + --nosyncinternaldates --delete2 --expunge2 + #--debugimap2 + can_send && sendtestmessage toto + $CMD_PERL_3xx ./imapsync \ + --host1 $HOST1 --user1 toto \ + --passfile1 ../../var/pass/secret.toto \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX --noauthmd5 \ + --nosyncinternaldates --delete2 --expunge2 + #--debugimap2 +} +# bug: +# $d=""; # no bug with $d=undef +# $imap2->append_string($h2_fold,$string, $h1_flags, $d); +# 3.25 idate : Sending: 16 APPEND INBOX () "16-Jul-2010 22:09:42 +0200" {428} +# 2.xx idate : Sending: 62 APPEND INBOX "16-Jul-2010 22:14:00 +0200" {428} +# 3.25 noidate: Sending: 16 APPEND INBOX () "" {428} # Fails: NO IMAP! +# 2.xx noidate: Sending: 62 APPEND INBOX {428} ll_idatefromheader() { if can_send; then @@ -935,9 +947,17 @@ ll_bigmail() { msw() { - sendtestmessage toto - scp imapsync Admin@192.168.68.77:'C:/msys/1.0/home/Admin/imapsync/imapsync' - ssh Admin@192.168.68.77 'C:/msys/1.0/home/Admin/imapsync/test.bat' + if can_send; then + sendtestmessage toto + fi + scp imapsync test.bat \ + ../../var/pass/secret.toto \ + ../../var/pass/secret.titi \ + ../../var/pass/secret.tata \ + Admin@c:'C:/msys/1.0/home/Admin/imapsync/' + + ssh Admin@c 'C:/msys/1.0/home/Admin/imapsync/test.bat' + scp Admin@c:'C:/msys/1.0/home/Admin/imapsync/imapsync.exe' . } @@ -1075,7 +1095,7 @@ big_transfert() --host2 $HOST2 --user2 tete@est.belle \ --passfile2 ../../var/pass/secret.tete \ --noauthmd5 \ - --fast --folder INBOX.Trash \ + --fast --folder INBOX.Junk \ --useheader Message-ID --useheader Received || \ true } @@ -1092,7 +1112,7 @@ big_transfert_sizes_only() --host2 $HOST2 --user2 tete@est.belle \ --passfile2 ../../var/pass/secret.tete \ --noauthmd5 \ - --justfoldersizes --folder INBOX.Trash || \ + --justfoldersizes --folder INBOX.Junk || \ true } date2=`date` @@ -1101,23 +1121,48 @@ big_transfert_sizes_only() -dprof() +dprof_justfoldersizes() { date1=`date` - { perl -d:DProf ./imapsync \ - --host1 louloutte --user1 gilles \ - --passfile1 ../../var/pass/secret \ - --host2 plume --user2 tete@est.belle \ + { $CMD_PERL -d:DProf ./imapsync \ + --host1 $HOST1 --user1 gilles@est.belle \ + --passfile1 ../../var/pass/secret.gilles_mbox \ + --host2 $HOST2 --user2 tete@est.belle \ --passfile2 ../../var/pass/secret.tete \ - --subscribed --foldersizes --noauthmd5 \ - --folder INBOX.Trash || \ + --noauthmd5 \ + --justfoldersizes --folder INBOX.Junk || \ true } date2=`date` echo3 "[$date1] [$date2]" - dprofpp tmon.out + mv tmon.out dprof_justfoldersizes_tmon.out + dprofpp -O 30 dprof_justfoldersizes_tmon.out + dprofpp -O 30 -I dprof_justfoldersizes_tmon.out } + +dprof_bigfolder() +{ + date1=`date` + { $CMD_PERL -d:DProf ./imapsync \ + --host1 $HOST1 --user1 gilles@est.belle \ + --passfile1 ../../var/pass/secret.gilles_mbox \ + --host2 $HOST2 --user2 tete@est.belle \ + --passfile2 ../../var/pass/secret.tete \ + --noauthmd5 \ + --nofoldersizes --folder INBOX.15_imapsync.imapsync-list || \ + true + } + date2=`date` + echo3 "[$date1] [$date2]" + mv tmon.out dprof_bigfolder_tmon.out + dprofpp -O 30 dprof_bigfolder_tmon.out + dprofpp -O 30 -I dprof_bigfolder_tmon.out +} + + + + essnet_justconnect() { ./imapsync \ @@ -1293,6 +1338,7 @@ no_args option_version option_tests option_bad_delete2 +passwords_masked first_sync_dry first_sync locallocal @@ -1304,7 +1350,7 @@ ll_folder ll_buffersize ll_justfolders ll_prefix12 -ll_internaldate +ll_nosyncinternaldates ll_idatefromheader ll_folder_rev ll_subscribed