diff --git a/CREDITS b/CREDITS index 3fed3ae..881a2f5 100644 --- a/CREDITS +++ b/CREDITS @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: CREDITS,v 1.143 2010/08/15 11:18:09 gilles Exp gilles $ +# $Id: CREDITS,v 1.144 2010/09/06 01:08:41 gilles Exp gilles $ If you want to make a donation to the author, Gilles LAMIRAL: @@ -20,6 +20,15 @@ to remove one. I thank very much all of these people. +Kirk Ismay +Contributed by giving money 50 USD + +Fast-Serv Technologies +Contributed by giving money 10 USD + +Christian Leue +Contributed by giving money 25 USD + Tony Pearse Contributed by giving money 25 USD diff --git a/ChangeLog b/ChangeLog index e85d190..53083e3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,17 +1,42 @@ RCS file: RCS/imapsync,v Working file: imapsync -head: 1.344 +head: 1.350 branch: locks: strict - gilles: 1.344 + gilles: 1.350 access list: symbolic names: keyword substitution: kv -total revisions: 344; selected revisions: 344 +total revisions: 350; selected revisions: 350 description: ---------------------------- -revision 1.344 locked by: gilles; +revision 1.350 locked by: gilles; +date: 2010/09/06 01:05:09; author: gilles; state: Exp; lines: +33 -24 +Added --noreleasecheck option. +Added User-agent information (OS, perl version, Mail::IMAPClient version) +---------------------------- +revision 1.349 +date: 2010/09/06 00:25:03; author: gilles; state: Exp; lines: +71 -13 +Added new release checking +---------------------------- +revision 1.348 +date: 2010/09/05 21:28:35; author: gilles; state: Exp; lines: +16 -18 +Strong regex for month in good_date() +---------------------------- +revision 1.347 +date: 2010/09/05 21:12:10; author: gilles; state: Exp; lines: +61 -50 +Rewrote good_date and removed Date::Manip usage. +---------------------------- +revision 1.346 +date: 2010/09/05 16:28:18; author: gilles; state: Exp; lines: +34 -36 +Simplified default output. +---------------------------- +revision 1.345 +date: 2010/08/24 01:44:59; author: gilles; state: Exp; lines: +45 -45 +Better output when copying messages. +---------------------------- +revision 1.344 date: 2010/08/20 02:06:13; author: gilles; state: Exp; lines: +123 -49 Added function fetch_hash_2() allowing a list of uid to be fetched. Changed fetch_hash() calls by fetch_hash_2() calls diff --git a/Makefile b/Makefile index b051a2a..8f657ab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -# $Id: Makefile,v 1.36 2010/08/15 17:36:04 gilles Exp gilles $ +# $Id: Makefile,v 1.37 2010/08/24 01:46:36 gilles Exp gilles $ .PHONY: help usage all @@ -23,7 +23,7 @@ DEB_FILE=$(DIST_NAME).deb VERSION=$(shell perl -I./Mail-IMAPClient-2.2.9 ./imapsync --version) -all: ChangeLog README VERSION +all: ChangeLog README VERSION VERSION_EXE .PHONY: test tests testp testf test3xx @@ -69,9 +69,10 @@ ChangeLog: imapsync README: imapsync perldoc -t imapsync > README -VERSION: imapsync Makefile +VERSION: imapsync perl -I./Mail-IMAPClient-2.2.9 ./imapsync --version > VERSION + .PHONY: clean clean_tilde clean_test clean: clean_tilde clean_man @@ -130,10 +131,11 @@ clean_dist: .PHONY: lfo upload_lfo niouze_lfo niouze_fm public dosify_bat imapsync_cidone -upload_index: index.shtml +upload_index: index.shtml + rcsdiff index.shtml rsync -avH index.shtml \ - /home/gilles/public_html/www.linux-france.org/html/prj/imapsync/ - sh ~/memo/lfo-rsync + ../../public_html/www.linux-france.org/html/prj/imapsync/ + sh $(HOME)/memo/lfo-rsync .dosify_bat: build_exe.bat test_exe.bat test.bat unix2dos build_exe.bat test.bat test_exe.bat @@ -158,6 +160,7 @@ imapsync.exe: imapsync imapsync_cidone dosify_bat time ssh Admin@c 'C:/msys/1.0/home/Admin/imapsync/build_exe.bat' time ssh Admin@c 'C:/msys/1.0/home/Admin/imapsync/test_exe.bat' scp Admin@c:'C:/msys/1.0/home/Admin/imapsync/imapsync.exe' . + ssh Admin@c 'C:/msys/1.0/home/Admin/imapsync/imapsync.exe --version' > VERSION_EXE lfo: dist niouze_lfo upload_lfo diff --git a/README b/README index e31ce97..d6be304 100644 --- a/README +++ b/README @@ -3,7 +3,7 @@ NAME Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 36 different IMAP server softwares supported with success. - $Revision: 1.344 $ + $Revision: 1.350 $ SYNOPSIS To synchronise imap account "foo" on "imap.truc.org" to imap account @@ -325,13 +325,14 @@ IMAP SERVERS - SmarterMail, Smarter Mail 5.0 Enterprise. - SunONE Messaging server 5.2, 6.0 (SUN JES - Java Enterprise System) - Sun Java(tm) System Messaging Server 6.2-2.05, 6.2-7.05 + - Sun Messaging Server 6.3 - Surgemail 3.6f5-5 - UW-imap servers (imap-2000b) rijkkramer IMAP4rev1 2000.287 (RedHat uses UW like 2003.338rh), v12.264 Solaris 5.7 (OSI Approved) (http://www.washington.edu/imap/) - UW - QMail v2.1 - Imap part of TCP/IP suite of VMS 7.3.2 - - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5. + - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5, 6.x Please report to the author any success or bad story with imapsync and do not forget to mention the IMAP server software names and version on @@ -340,7 +341,7 @@ IMAP SERVERS are useful to know the softwares. Example: Host1 software:* OK louloutte Cyrus IMAP4 v1.5.19 server ready - Host2 software:* OK Courier-IMAP ready + Host2 software:* OK Courier-IMAP ready You can use option --justconnect to get those lines. Example: @@ -400,5 +401,5 @@ SIMILAR SOFTWARES Feedback (good or bad) will often be welcome. - $Id: imapsync,v 1.344 2010/08/20 02:06:13 gilles Exp gilles $ + $Id: imapsync,v 1.350 2010/09/06 01:05:09 gilles Exp gilles $ diff --git a/TIME b/TIME index 5ee82a9..da48536 100644 --- a/TIME +++ b/TIME @@ -1,3 +1,7 @@ +360 Date::Manip away. good_date() rewriting. release check on lfo +210 Better output when copying messages. Profiling memory. + 80 Gmail efficiency. Wanted! on homepage, email on list. +300 Large mailbox --maxage. Upload 1.344. imapsync.exe 150 Various. Upload 1.342 75 Fix. capability issue after authenticate. 240 Various diff --git a/TODO b/TODO index d94e1a1..4463230 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: TODO,v 1.83 2010/08/20 02:12:05 gilles Exp gilles $ +# $Id: TODO,v 1.85 2010/09/06 01:08:14 gilles Exp gilles $ TODO file for imapsync ---------------------- @@ -21,6 +21,18 @@ Evaluate http://www.rackspace.com/apps/email_hosting/migrations http://www.yippiemove.com/ +Evaluate memory consumption with (or better): +print qx{ ps o pid,pcpu,comm,vsz,rss,size $$ }, "\n" +Search memory leaks with +Test-Weaken Test-Memory-Cycle Devel-Cycle Devel-Leak Test-Weaken +sh -x tests.sh ll_bigmail +is a good candidate to stress memory. + +Suggestion: it's very difficult to track down messages which are behaving +funny during the sync. It would be great - and presumably easy to code - +to have an option to have imapsync display e.g. the subject of an +e-mail when it gets synced, rather than just the message ID and the date/time. + Add --noauthmd51 --noauthmd52 to permit noauthmd5 by host diff --git a/VERSION b/VERSION index 25f7daf..0b73934 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.344 +1.350 diff --git a/VERSION_EXE b/VERSION_EXE new file mode 100644 index 0000000..bcdbccd --- /dev/null +++ b/VERSION_EXE @@ -0,0 +1 @@ +1.350 diff --git a/build_exe.bat b/build_exe.bat index 645b8e7..f100900 100755 --- a/build_exe.bat +++ b/build_exe.bat @@ -1,11 +1,10 @@ -REM $Id: test.bat,v 1.6 2010/08/15 11:10:49 gilles Exp gilles $ +REM $Id: build_exe.bat,v 1.3 2010/09/06 02:16:24 gilles Exp gilles $ echo Building imapsync.exe cd C:\msys\1.0\home\Admin\imapsync -perl -mMail::IMAPClient -mDigest::MD5 -mTerm::ReadKey -mIO::Socket::SSL -mDate::Manip -mFile::Spec -mDigest::HMAC_MD5 -e '' +perl -mMail::IMAPClient -mDigest::MD5 -mTerm::ReadKey -mIO::Socket::SSL -mFile::Spec -mDigest::HMAC_MD5 -e '' -REM -M Date::Manip 6.xx buggy? pp -o imapsync.exe -M Term::ReadKey -M IO::Socket::SSL -M Digest::HMAC_MD5 imapsync echo Done building imapsync.exe diff --git a/freshmeat_submition.inp b/freshmeat_submition.inp index dcb7de5..4268236 100644 --- a/freshmeat_submition.inp +++ b/freshmeat_submition.inp @@ -4,17 +4,17 @@ #RELEASE_FOCUS="Documentation" #RELEASE_FOCUS="Code cleanup" #RELEASE_FOCUS="Minor feature enhancements" -#RELEASE_FOCUS="Major feature enhancements" -RELEASE_FOCUS="Minor bugfixes" +RELEASE_FOCUS="Major feature enhancements" +#RELEASE_FOCUS="Minor bugfixes" #RELEASE_FOCUS="Major bugfixes" #RELEASE_FOCUS="Minor security fixes" #RELEASE_FOCUS="Major security fixes" - #TEXT_BODY="Syntax cleanup" #TEXT_BODY="Updated documentation" -TEXT_BODY="Bug fixes. - -Many thanks to the freshmeat folk that correct my bad and poorly English ! +TEXT_BODY=" +Since 1.350: +Bug fixes. +Many thanks to the freshmeat folk that correct my bad and poorly English! " diff --git a/freshmeat_submition.json b/freshmeat_submition.json new file mode 100644 index 0000000..8b2c6e6 --- /dev/null +++ b/freshmeat_submition.json @@ -0,0 +1,31 @@ +{ + "release": { + "tag_list": "Major feature enhancements", + "version": "1.350", + "hidden_from_frontpage": false, + "changelog": " +Since 1.286: +Added --noreleasecheck option. +Added new release checking. +Removed Date::Manip dependancy. +Better output when copying messages. +More effiscient with large mailboxes. +Clarity: print capability after authenticated state. +Duplicate messages on host2 are now deleted with --delete2 +--skipsize turned on by default. +Usability fix: examples with --password1 instead of --passfile1 +Added --debugimap1 --debugimap2 to permit imap outpout with only one host. +Added reconnect statistics. +Added reconnect behavior with Mail::IMAPClient 2.2.9 +Added statistic about messages deleted. +Added statistic about average bandwith rate. +Flags are now exactly synced from host1 to host2 +(Previous releases just added flags). +Added TLSv1 support. +Filter flags sync with the list given by PERMANENTFLAGS on --host2 +and bug fixes. +Many thanks to the freshmeat folk that correct my bad and poorly English! +" + } +} + diff --git a/imapsync b/imapsync index 4710342..37bcfe5 100755 --- a/imapsync +++ b/imapsync @@ -6,6 +6,7 @@ # main program # global variables initialisation # default values +# folder loop # subroutines # IMAPClient 2.2.9 overrides # IMAPClient 2.2.9 3.xx ads @@ -19,7 +20,7 @@ tool. Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 36 different IMAP server softwares supported with success. -$Revision: 1.344 $ +$Revision: 1.350 $ =head1 SYNOPSIS @@ -372,13 +373,14 @@ Success stories reported with the following 36 imap servers - SmarterMail, Smarter Mail 5.0 Enterprise. - SunONE Messaging server 5.2, 6.0 (SUN JES - Java Enterprise System) - Sun Java(tm) System Messaging Server 6.2-2.05, 6.2-7.05 + - Sun Messaging Server 6.3 - Surgemail 3.6f5-5 - UW-imap servers (imap-2000b) rijkkramer IMAP4rev1 2000.287 (RedHat uses UW like 2003.338rh), v12.264 Solaris 5.7 (OSI Approved) (http://www.washington.edu/imap/) - UW - QMail v2.1 - Imap part of TCP/IP suite of VMS 7.3.2 - - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5. + - Zimbra-IMAP 3.0.1 GA 160, 3.1.0 Build 279, 4.0.5, 4.5.2, 4.5.6, 5.5, 6.x Please report to the author any success or bad story with imapsync and do not forget to mention the IMAP server @@ -388,7 +390,7 @@ report the two lines at the begining of the output if they are useful to know the softwares. Example: Host1 software:* OK louloutte Cyrus IMAP4 v1.5.19 server ready - Host2 software:* OK Courier-IMAP ready + Host2 software:* OK Courier-IMAP ready You can use option --justconnect to get those lines. Example: @@ -469,7 +471,7 @@ Entries for imapsync: Feedback (good or bad) will often be welcome. -$Id: imapsync,v 1.344 2010/08/20 02:06:13 gilles Exp gilles $ +$Id: imapsync,v 1.350 2010/09/06 01:05:09 gilles Exp gilles $ =cut @@ -551,13 +553,14 @@ my( $tests, $test_builder, $tests_debug, $allow3xx, $justlogin, $tmpdir, + $releasecheck, ); # main program # global variables initialisation -$rcs = '$Id: imapsync,v 1.344 2010/08/20 02:06:13 gilles Exp gilles $ '; +$rcs = '$Id: imapsync,v 1.350 2010/09/06 01:05:09 gilles Exp gilles $ '; $total_bytes_transferred = 0; $total_bytes_skipped = 0; @@ -581,6 +584,9 @@ my @argv_copy = @ARGV; get_options(); +$releasecheck = defined($releasecheck) ? $releasecheck : 1; +my $warn_release = ($releasecheck) ? check_last_release() : ''; + # default values $tmpdir ||= File::Spec->tmpdir(); @@ -589,9 +595,6 @@ $pidfile ||= $tmpdir . '/imapsync.pid'; # allow Mail::IMAPClient 3.0.xx by default $allow3xx = defined($allow3xx) ? $allow3xx : 1; -# Does not use Date::Manip by default: buggy 5.x vs 6.x and slow -$usedatemanip = defined($usedatemanip) ? $usedatemanip : 0; - print banner_imapsync(@argv_copy); print "Temp directory is $tmpdir\n"; @@ -633,21 +636,6 @@ sub connect_imap { or die_clean("Can not open imap connection on [$host]: $@\n"); } -sub localhost_info { - - my($infos) = join("", - "Here is a [$OSNAME] system (", - join(" ", - uname(), - ), - ")\n", - "With perl ", - sprintf("%vd", $PERL_VERSION), - " Mail::IMAPClient $Mail::IMAPClient::VERSION", - ); - return($infos); - -} if ($justconnect) { justconnect(); @@ -672,32 +660,6 @@ if ($syncinternaldates) { print "Turned OFF syncinternaldates\n"; } -if ($syncinternaldates || $idatefromheader) { - - # Date::Manip is an ugly module: it exits (confess) for reading an unset value - # I should write a bug report but I'm too lazy. - - no warnings 'redefine'; - local *Carp::confess = sub { return undef; }; - require Date::Manip; - Date::Manip->import(qw(ParseDate UnixDate Date_Init Date_TimeZone)); - - if ($OSNAME eq "MSWin32") { - # It seems that local *Carp does not work on win32 - my $TZ = $ENV{TZ} || 'GMT'; - Date_Init("TZ=$TZ"); - print "TimeZone: [", Date_TimeZone(), "]\n"; - }else{ - #print "Date_init: [", join(" ",Date_Init()), "]\n"; - print "TimeZone:[", Date_TimeZone(), "]\n"; - if (not (Date_TimeZone())) { - warn "TimeZone not defined, setting it to GMT"; - Date_Init("TZ=GMT"); - print "TimeZone: [", Date_TimeZone(), "]\n"; - } - } -} - if(defined($authmd5) and not($authmd5)) { $authmech1 ||= 'LOGIN'; @@ -1167,11 +1129,11 @@ sub foldersizes { my $tot = 0; my $tmess = 0; my @folders = @{$folders_r}; - print "++++ Calculating sizes ++++\n"; + print "++++ Calculating sizes\n"; foreach my $folder (@folders) { my $stot = 0; my $smess = 0; - printf("$side Folder %-35s", "[$folder]"); + printf("$side folder %-35s", "[$folder]"); unless($imap->exists($folder)) { print("does not exist yet\n"); next; @@ -1219,8 +1181,6 @@ if ($foldersizes) { } - - sub timenext { my ($timenow, $timerel); # $timebefore is global, beurk ! @@ -1242,7 +1202,7 @@ foreach my $folder (@h2_folders_list) { } print - "++++ Listing folders ++++\n", + "++++ Listing folders\n", "Host1 folders list:\n", map("[$_]\n",@h1_folders),"\n", "Host2 folders list:\n", map("[$_]\n",@h2_folders_list),"\n"; @@ -1516,23 +1476,21 @@ sub flags_filter { return($flags_out); } -print "++++ Looping on each folder ++++\n"; -#sleep 10; +# folder loop +print "++++ Looping on each folder\n"; + FOLDER: foreach my $h1_fold (@h1_folders) { last FOLDER if $imap1->IsUnconnected(); last FOLDER if $imap2->IsUnconnected(); - my $h2_fold; - print "Host1 Folder [$h1_fold]\n"; - $h2_fold = imap2_folder_name($h1_fold); - print "Host2 Folder [$h2_fold]\n"; - + my $h2_fold = imap2_folder_name($h1_fold); + printf("%-35s -> %-35s\n", "[$h1_fold]", "[$h2_fold]"); unless ($imap1->select($h1_fold)) { warn - "Host1 Folder $h1_fold: Could not select: ", + "Host1 folder $h1_fold: Could not select: ", $imap1->LastError, "\n"; $nb_errors++; next FOLDER; @@ -1597,7 +1555,7 @@ FOLDER: foreach my $h1_fold (@h1_folders) { my %h1_hash = (); my %h2_hash = (); - print "++++ Host1 [$h1_fold] parsing headers ++++\n"; + $debug and print "Host1 folder [$h1_fold] parsing headers\n"; last FOLDER if $imap1->IsUnconnected(); last FOLDER if $imap2->IsUnconnected(); @@ -1612,7 +1570,7 @@ FOLDER: foreach my $h1_fold (@h1_folders) { $debug and print "Time fir: ", timenext(), " s\n"; unless ($h1_fir_ref) { warn - "Host1 Folder $h1_fold: Could not fetch_hash_2 ", + "Host1 folder $h1_fold: Could not fetch_hash_2 ", scalar(@h1_msgs), " msgs: ", $imap1->LastError, "\n"; $nb_errors++; next FOLDER; @@ -1641,7 +1599,7 @@ FOLDER: foreach my $h1_fold (@h1_folders) { } $debug and print "Time parsing headers on host1: ", timenext(), " s\n"; - print "++++ Host2 [$h2_fold] parsing headers ++++\n"; + $debug and print "Host2 folder [$h2_fold] parsing headers\n"; my ($h2_heads_ref, $h2_fir_ref) = ({}, {}); $h2_heads_ref = $imap2->parse_headers([@h2_msgs], @useheader) if (@h2_msgs); @@ -1671,8 +1629,8 @@ FOLDER: foreach my $h1_fold (@h1_folders) { } $debug and print "Time parsing headers on host2: ", timenext(), " s\n"; - print "++++ Verifying [$h1_fold] -> [$h2_fold] ++++\n"; - # messages in host1 that are not good in host2 + $debug and print "++++ Verifying [$h1_fold] -> [$h2_fold]\n"; + # messages in host1 that are not in host2 my @h1_hash_keys_sorted_by_uid = sort {$h1_hash{$a}{'m'} <=> $h1_hash{$b}{'m'}} keys(%h1_hash); @@ -1691,7 +1649,7 @@ FOLDER: foreach my $h1_fold (@h1_folders) { my $h2_msg = $h2_hash{$m_id}{'m'}; my $h2_flags = $h2_hash{$m_id}{'F'} || ""; my $isdel = $h2_flags =~ /\B\\Deleted\b/ ? 1 : 0; - print "deleting message [$m_id] #$h2_msg in host2 folder $h2_fold\n" + print "msg $h2_fold/$h2_msg deleted on host2 [$m_id]\n" if ! $isdel; push(@h2_expunge, $h2_msg) if $uidexpunge2; unless ($dry or $isdel) { @@ -1701,7 +1659,7 @@ FOLDER: foreach my $h1_fold (@h1_folders) { } } foreach my $h2_msg (@h2_msgs_duplicate) { - print "deleting message [duplicate] #$h2_msg in host2 folder $h2_fold\n"; + print "msg $h2_fold/$h2_msg deleted [duplicate] on host2\n"; push(@h2_expunge, $h2_msg) if $uidexpunge2; unless ($dry) { $imap2->delete_message($h2_msg); @@ -1720,28 +1678,26 @@ FOLDER: foreach my $h1_fold (@h1_folders) { } MESS: foreach my $m_id (@h1_hash_keys_sorted_by_uid) { - my $h1_size = $h1_hash{$m_id}{'s'}; - my $h1_msg = $h1_hash{$m_id}{'m'}; + my $h1_size = $h1_hash{$m_id}{'s'}; + my $h1_msg = $h1_hash{$m_id}{'m'}; my $h1_idate = $h1_hash{$m_id}{'D'}; if (defined $maxsize and $h1_size > $maxsize) { - print "+ Skipping msg #$h1_msg:$h1_size in host1 folder $h1_fold (exceeds maxsize limit $maxsize bytes)\n"; + print "msg $h1_fold/$h1_msg skipping ($h1_size exceeds maxsize limit $maxsize bytes)\n"; $total_bytes_skipped += $h1_size; $nb_msg_skipped += 1; next MESS; } - $debug and print "+ key $m_id #$h1_msg\n"; unless (exists($h2_hash{$m_id})) { - print "+ NO msg #$h1_msg [$m_id] in $h2_fold\n"; # copy - print "+ Copying msg #$h1_msg:$h1_size to folder $h2_fold\n"; + $debug and print "msg $h1_fold/$h1_msg copying to $h2_fold\n"; last FOLDER if $imap1->IsUnconnected(); - last FOLDER if $imap2->IsUnconnected(); + last FOLDER if $imap2->IsUnconnected(); my $string; $string = $imap1->message_string($h1_msg); unless (defined($string)) { warn - "Could not fetch message #$h1_msg from $h1_fold: ", + "- msg $h1_fold/$h1_msg could not fetch [$m_id $h1_size]: ", $imap1->LastError, "\n"; $nb_errors++; $total_bytes_error += $h1_size; @@ -1851,28 +1807,20 @@ Bye.' "F message content begin next line\n", $string, "F message content ended on previous line\n", "=" x 80, "\n"; - my $d = ""; + my $h1_date = ""; if ($syncinternaldates) { - $d = $h1_idate; - $debug and print "internal date from 1: [$d]\n"; - $d = good_date($d); - $debug and print "internal date from 1: [$d] (fixed)\n"; + $h1_date = $h1_idate; + $debug and print "internal date from host1: [$h1_date]\n"; + $h1_date = good_date($h1_date); + $debug and print "internal date from host1: [$h1_date] (fixed)\n"; } if ($idatefromheader) { - $d = $imap1->get_header($h1_msg,"Date"); - $debug and print "header date from 1: [$d]\n"; - $d = good_date($d); - $debug and print "header date from 1: [$d] (fixed)\n"; - } - - sub good_date { - my ($d) = @_; - return($d) if (! $usedatemanip); - $d = UnixDate(ParseDate($d), "%d-%b-%Y %H:%M:%S %z"); - $d = "\"$d\""; - return($d); + $h1_date = $imap1->get_header($h1_msg,"Date"); + $debug and print "header date from host1: [$h1_date]\n"; + $h1_date = good_date($h1_date); + $debug and print "header date from host1: [$h1_date] (fixed)\n"; } my $h1_flags = $h1_hash{$m_id}{'F'} || ""; @@ -1883,24 +1831,24 @@ Bye.' $h1_flags = flags_filter($h1_flags, $permanentflags2) if ($permanentflags2); my $new_id; - print "flags & date from: [$h1_flags][$d]\n"; + $debug and print "msg $h1_fold/$h1_msg date [$h1_date] flags [$h1_flags] size [$h1_size]\n"; last FOLDER if $imap1->IsUnconnected(); - last FOLDER if $imap2->IsUnconnected(); - $d = undef if ($d eq ""); + last FOLDER if $imap2->IsUnconnected(); + $h1_date = undef if ($h1_date eq ""); unless ($dry) { if ($OSNAME eq "MSWin32") { - $new_id = $imap2->append_string($h2_fold,$string, $h1_flags, $d); + $new_id = $imap2->append_string($h2_fold, $string, $h1_flags, $h1_date); } else { # just back to append_string since append_file 3.05 does not work. #$new_id = $imap2->append_file($h2_fold, $message_file, "", $h1_flags, $d); # append_string 3.05 does not work too some times with $d unset. - $new_id = $imap2->append_string($h2_fold,$string, $h1_flags, $d); + $new_id = $imap2->append_string($h2_fold, $string, $h1_flags, $h1_date); } unless($new_id){ no warnings 'uninitialized'; - warn "Couldn't append msg #$h1_msg (Subject:[". + warn "- msg $h1_fold/$h1_msg couldn't append (Subject:[". $imap1->subject($h1_msg)."]) to folder $h2_fold: ", $imap2->LastError, "\n"; $nb_errors++; @@ -1911,11 +1859,11 @@ Bye.' # good # $new_id is an id if the IMAP server has the # UIDPLUS capability else just a ref - print "Copied msg id [$h1_msg] to folder $h2_fold msg id [$new_id]\n"; + print "msg $h1_fold/$h1_msg copied to $h2_fold/$new_id\n"; $total_bytes_transferred += $h1_size; $nb_msg_transferred += 1; if($delete) { - print "Deleting msg #$h1_msg on host1 folder $h1_fold\n"; + print "msg $h1_fold/$h1_msg deleted on host1\n"; unless($dry) { $imap1->delete_message($h1_msg); $h1_nb_msg_deleted += 1; @@ -1933,7 +1881,10 @@ Bye.' next MESS; } else{ - $debug and print "Message id [$m_id] found in t:$h2_fold\n"; + #my $h2_size = $h2_hash{$m_id}{'s'}; + my $h2_msg = $h2_hash{$m_id}{'m'}; + #my $h2_idate = $h2_hash{$m_id}{'D'}; + $debug and print "msg $h1_fold/$h1_msg equals $h2_fold/$h2_msg\n"; $total_bytes_skipped += $h1_size; $nb_msg_skipped += 1; } @@ -1958,7 +1909,7 @@ Bye.' my @h2_flags = sort split(' ', $h2_flags ); my $diff = compare_lists(\@h1_flags, \@h2_flags); - $diff and $debug and print "Replacing h2 flags($h2_flags) with h1 flags($h1_flags) on msg #$h2_msg in $h2_fold\n"; + $diff and $debug and print "msg $h2_fold/$h2_msg replacing h2 flags($h2_flags) with h1 flags($h1_flags)\n"; # This sets flags so flags can be removed with this # When you remove a \Seen flag on host1 you want to it @@ -1966,8 +1917,7 @@ Bye.' # we need most of the time. if (!$dry and $diff and !$imap2->store($h2_msg, "FLAGS.SILENT (@h1_flags)") ) { - warn "Could not add flags @h1_flags", - " on msg #$h2_msg in $h2_fold: ", + warn "- msg $h2_fold/$h2_msg could not add flags @h1_flags", $imap2->LastError, "\n"; #$nb_errors++; } @@ -1990,21 +1940,21 @@ Bye.' "host2 internal date: $h2_idate\n"; #unless ($h1_idate eq $h2_idate) { - # print "!!! Dates differ !!!\n"; + # print "!!! Dates differs !!!\n"; #} }; unless ($skipsize or ($h1_size == $h2_size)) { # Bad size print - "Message $m_id SZ_BAD f:$h1_msg:$h1_size t:$h2_msg:$h2_size\n"; + "- msg $h1_fold/$h1_msg size diff $h1_size != $h2_size $h2_fold/$h2_msg\n"; $nb_errors++; } else { # Good $debug and print - "Message $m_id SZ_GOOD f:$h1_msg:$h1_size t:$h2_msg:$h2_size\n"; + "msg $h1_fold/$h1_msg sizes ok $h1_size <=> $h2_size $h2_fold/$h2_msg\n"; if($delete) { - print "Deleting msg #$h1_msg on host1 folder $h1_fold\n"; + print "msg $h1_fold/$h1_msg deleted on host1\n"; unless($dry) { $imap1->delete_message($h1_msg); $h1_nb_msg_deleted += 1; @@ -2022,10 +1972,10 @@ Bye.' unless($dry) { $imap2->expunge() }; } -print "Time: ", timenext(), " s\n"; +$debug and print "Time: ", timenext(), " s\n"; } -print "++++ End looping on each folder ++++\n"; +print "++++ End looping on each folder\n"; # FOLDER loop is exited any time a connection is lost be sure to log it! @@ -2085,14 +2035,6 @@ exit_clean(0); # subroutines -sub imapsync_version { - my $rcs = '$Id: imapsync,v 1.344 2010/08/20 02:06:13 gilles Exp gilles $ '; - $rcs =~ m/,v (\d+\.\d+)/; - my $VERSION = ($1) ? $1: "UNKNOWN"; - return($VERSION); -} - - sub check_lib_version { $debug and print "IMAPClient $Mail::IMAPClient::VERSION\n"; if ($Mail::IMAPClient::VERSION eq '2.2.9') { @@ -2119,8 +2061,7 @@ IO::Socket IO::Socket::SSL Digest::MD5 Digest::HMAC_MD5 -Term::ReadKey -Date::Manip)) +Term::ReadKey)) { my $v = "?"; @@ -2170,8 +2111,8 @@ sub banner_imapsync { my @argv_copy = @_; my $banner_imapsync = join("", '$RCSfile: imapsync,v $ ', - '$Revision: 1.344 $ ', - '$Date: 2010/08/20 02:06:13 $ ', + '$Revision: 1.350 $ ', + '$Date: 2010/09/06 01:05:09 $ ', "\n",localhost_info(), "\n", "Command line used:\n", "$0 ", command_line_nopassword(@argv_copy), "\n", @@ -2274,7 +2215,7 @@ sub select_msgs { } sub stats { - print "++++ Statistics ++++\n"; + print "++++ Statistics\n"; print "Transfer time : $timediff sec\n"; print "Messages transferred : $nb_msg_transferred "; print "(could be $nb_msg_skipped_dry_mode without dry mode)" if ($dry); @@ -2297,6 +2238,8 @@ sub stats { print "Reconnections to host1 : $host1_reconnect_count\n"; print "Reconnections to host2 : $host2_reconnect_count\n"; print "Detected $nb_errors errors\n\n"; + + print $warn_release, "\n"; print thank_author(); } @@ -2350,7 +2293,6 @@ sub get_options { "delete2!" => \$delete2, "syncinternaldates!" => \$syncinternaldates, "idatefromheader!" => \$idatefromheader, - "usedatemanip!" => \$usedatemanip, "syncacls!" => \$syncacls, "maxsize=i" => \$maxsize, "maxage=i" => \$maxage, @@ -2397,7 +2339,7 @@ sub get_options { "justlogin!" => \$justlogin, "tmpdir=s" => \$tmpdir, "pidfile=s" => \$pidfile, - + "releasecheck!" => \$releasecheck, ); $debug and print "get options: [$opt_ret]\n"; @@ -2431,7 +2373,6 @@ sub get_options { sub load_modules { require IO::Socket::SSL if ($ssl1 or $ssl2 or $tls1 or $tls2); - require Date::Manip if ($syncinternaldates || $idatefromheader) ; require Term::ReadKey if ( ((not($password1 or $passfile1)) @@ -2540,10 +2481,92 @@ sub string_to_file { } +sub check_last_release { + + my $public_release = not_long('imapsync_version_lfo'); + return('') if ($public_release eq 'unknown'); + + my $imapsync_here = imapsync_version(); + + if ($public_release > $imapsync_here) { + return("New imapsync release $public_release available"); + }else{ + return("This current imapsync is up to date"); + } +} + +sub imapsync_version { + my $rcs = '$Id: imapsync,v 1.350 2010/09/06 01:05:09 gilles Exp gilles $ '; + $rcs =~ m/,v (\d+\.\d+)/; + my $VERSION = ($1) ? $1: "UNKNOWN"; + return($VERSION); +} + + +sub imapsync_version_lfo { + + my $local_version = imapsync_version(); + my $agent_info = "$OSNAME system, perl $PERL_VERSION, Mail::IMAPClient $Mail::IMAPClient::VERSION"; + my $sock = new IO::Socket::INET ( + PeerAddr => 'linux-france.org', + PeerPort => '80', + Proto => 'tcp'); + return('unknown') if not $sock; + print $sock + "GET /prj/imapsync/VERSION HTTP/1.0\n", + "User-Agent: imapsync/$local_version ($agent_info)\n", + "Host: www.linux-france.org\n\n"; + my @line = <$sock>; + close($sock); + my $last_release = $line[-1]; + chomp($last_release); + return($last_release); +} + +sub not_long { + + my ($func) = @_; + my $val; + eval { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm 3; + #print $func, "\n"; + { + no strict "refs"; + $val = &$func(); + } + alarm 0; + }; + if ($@) { + # timed out + return('unknown') unless $@ eq "alarm\n"; # propagate unexpected errors + + }else { + # didn't + return($val); + } +} + +sub localhost_info { + + my($infos) = join("", + "Here is a [$OSNAME] system (", + join(" ", + uname(), + ), + ")\n", + "With perl ", + sprintf("%vd", $PERL_VERSION), + " Mail::IMAPClient $Mail::IMAPClient::VERSION", + ); + return($infos); + +} sub usage { my $localhost_info = localhost_info(); my $thank = thank_author(); + my $warn_release = check_last_release(); print < return same string + return($d); + } + + $d = qq("$d"); + return($d); +} + +sub tests_good_date { + + ok('' eq good_date(), 'good_date no arg'); + ok('"24-Aug-2010 16:00:00 +0200"' eq good_date('24-Aug-2010 16:00:00 +0200'), 'good_date internal 2digit zone'); + ok('"24-Aug-2010 16:00:00"' eq good_date('24-Aug-2010 16:00:00'), 'good_date internal 2digit no zone'); + ok('"01-Sep-2010 16:00:00 +0200"' eq good_date( '1-Sep-2010 16:00:00 +0200'), 'good_date internal SP 1digit'); + ok('"24-Aug-2010 16:00:00 +0200"' eq good_date('Tue, 24 Aug 2010 16:00:00 +0200'), 'good_date header 2digit zone'); + ok('"01-Sep-2010 16:00:00"' eq good_date('Wed, 1 Sep 2010 16:00:00'), 'good_date header SP 1digit zone'); + ok('"01-Sep-2010 16:00:00 +0200"' eq good_date('Wed, 1 Sep 2010 16:00:00 +0200'), 'good_date header SP 1digit zone'); + ok('"01-Sep-2010 16:00:00 +0200"' eq good_date('Wed, 1 Sep 2010 16:00:00 +0200 (CEST)'), 'good_date header SP 1digit zone'); + +} + sub tests_debug { SKIP: { skip "No test in normal run" if (not $tests_debug); - tests_regexmess(); + tests_good_date(); } } @@ -2727,6 +2800,7 @@ sub tests { tests_flags_filter(); tests_imap2_folder_name(); tests_command_line_nopassword(); + tests_good_date(); } } diff --git a/index.shtml b/index.shtml index 923bda6..a8f015d 100644 --- a/index.shtml +++ b/index.shtml @@ -5,7 +5,7 @@ imapsync <!--#exec cmd="cat VERSION" --> - + @@ -32,8 +32,8 @@ where the user plays independently on both sides. Use offlineimap (written by John Goerzen) for this purpose.

- -

AUTHOR

+ +

AUTHOR

Gilles LAMIRAL
Email: lamiral@linux-france.org

@@ -52,10 +52,10 @@ where the user plays independently on both sides. Use offlineimap for feedback.

- -

imapsync donation

-

Happy with this free, open and gratis software?

-

Help the author to maintain imapsync and support users:

+ +

imapsync call for donation

+

Are you happy with this free, open and gratis software?

+

Then you can help the author to maintain imapsync and support happy (or unhappy) users: YOU!

@@ -73,8 +73,8 @@ Or offer him a book on his Thanks in advance!

- -

Latest release + +

Latest release ()

@@ -83,18 +83,21 @@ Or offer him a book on his

imapsync source download

-Standalone imapsync executable for win32, +

Standalone +imapsync executable for win32, thanks to Strawberry Perl 5.12 and Par::Packed module.
-(imapsync.exe built time is ) +(imapsync.exe built time is )

imapsync installation

+ +

README

Frequently Asked Questions

- -

MAILING-LIST

+ +

MAILING-LIST

The public mailing-list may be the best way to get support.
@@ -133,8 +136,11 @@ thanks to Strawberry Perl 5.12 and Par::Packed module.
Thank you for your participation!

- -

TODO

+ +

+ +WANTED

+

I code new features for free when I have time and when I find it useful.
If you really want a feature you can donate money and I'll code it.
@@ -143,14 +149,15 @@ If you really want a feature you can donate money and I'll code it.

Some features and their time/money to be done evaluation

- - - - - - - - + + + + + + + + +
DONEFeature Time guessedTime spentMoney receivedMoney needed
NoSpeedup 50% 10 hours 80 min 10 $ 300 $
NoBackup to files 8 hours 60 min 0 $ 240 $
No--deletefolder2 3 hours 30 min 0 $ 90 $
NoNTLM auth 3 hours 60 min 0 $ 90 $
YesWin32 imapsync.exe 8 hours 520 min 0 $ 240 $
YesFix capability changes 1 hour 80 min 0 $ 30 $
YesLarge mailbox --maxage 4 hours 270 min 0 $ 120 $
DONEFeature Time guessedTime spentMoney receivedMoney needed
NoEfficient Gmail backup 8 hours 80 min 0 $ 240 $
NoSpeedup 50% 10 hours 80 min 10 $ 300 $
NoBackup to files 8 hours 60 min 0 $ 240 $
No--deletefolder2 3 hours 30 min 0 $ 90 $
NoNTLM auth 3 hours 60 min 0 $ 90 $
YesWin32 imapsync.exe 8 hours 520 min 0 $ 240 $
YesFix capability changes 1 hour 80 min 0 $ 30 $
YesLarge mailbox --maxage 4 hours 270 min 0 $ 120 $

COPYING

@@ -168,7 +175,7 @@ If you really want a feature you can donate money and I'll code it.
This document last modified
-$Id: index.shtml,v 1.20 2010/08/20 03:14:54 gilles Exp gilles $ +$Id: index.shtml,v 1.22 2010/08/21 13:39:35 gilles Exp gilles $

diff --git a/learn/io_socket_get b/learn/io_socket_get new file mode 100644 index 0000000..e7037f6 --- /dev/null +++ b/learn/io_socket_get @@ -0,0 +1,48 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use IO::Socket; + +sub last_release { + my $sock = new IO::Socket::INET ( + PeerAddr => 'linux-france.org', + PeerPort => '80', + Proto => 'tcp'); + return('unknown') if not $sock; + print $sock + "GET /prj/imapsync/VERSION HTTP/1.0\n", + "Host: www.linux-france.org\n\n"; + my @line = <$sock>; + close($sock); + my $last_release = $line[-1]; + chomp($last_release); + return($last_release); +} + +sub not_long { + + my ($func) = @_; + my $val; + eval { + local $SIG{ALRM} = sub { die "alarm\n" }; + alarm 3; + #print $func, "\n"; + { + no strict "refs"; + $val = &$func(); + } + alarm 0; + }; + if ($@) { + # timed out + return('unknown') unless $@ eq "alarm\n"; # propagate unexpected errors + + }else { + # didn't + return($val); + } +} + +print last_release(), "\n"; +print not_long('last_release'), "\n"; diff --git a/memo b/memo index 6283f15..177e9b2 100644 --- a/memo +++ b/memo @@ -42,8 +42,7 @@ fm_init() { software_version NEWS_FILE_FM="./freshmeat_submition" NEWS_FILE_FM_INP=${NEWS_FILE_FM}.inp -NEWS_FILE_FM_OUT=${NEWS_FILE_FM}.out - +NEWS_FILE_FM_OUT=${NEWS_FILE_FM}.json } fm_read_param() { @@ -58,14 +57,14 @@ fm_read_announce() { fm_read_param cat << EOF -Project: imapsync -Version: $VERSION -Release-Focus: $RELEASE_FOCUS -Hide: N -Home-Page-URL: http://www.linux-france.org/prj/imapsync/ -Gzipped-Tar-URL: http://www.linux-france.org/prj/imapsync/dist/ - -$TEXT_BODY +{ + "release": { + "tag_list": "stable, $RELEASE_FOCUS", + "version": "$VERSION", + "hidden_from_frontpage": false, + "changelog": "$TEXT_BODY" + } +} EOF } @@ -82,7 +81,10 @@ fm_announce() { fi fm_read_announce > $NEWS_FILE_FM_OUT - freshmeat-submit < $NEWS_FILE_FM_OUT + curl -X PUT -d @../../var/pass/secret.freshmeat -d @$NEWS_FILE_FM_OUT \ + -H "Content-Type: application/json" \ + http://freshmeat.net/projects/imapsync.json + fi } diff --git a/test_exe.bat b/test_exe.bat index 864ca2b..99adbae 100755 --- a/test_exe.bat +++ b/test_exe.bat @@ -3,8 +3,6 @@ cd C:\msys\1.0\home\Admin\imapsync perl -mMail::IMAPClient -mDigest::MD5 -mTerm::ReadKey -mIO::Socket::SSL -mFile::Spec -mDigest::HMAC_MD5 -e '' -set TZ="GMT" - .\imapsync.exe --host1 l --user1 toto --passfile1 secret.toto --host2 l --user2 titi --passfile2 secret.titi --noauthmd5 --delete2 --expunge2 .\imapsync.exe --host1 l --user1 tata --passfile1 secret.tata --host2 l --user2 titi --passfile2 secret.titi --noauthmd5 --delete2 --expunge2 --folder INBOX diff --git a/tests.sh b/tests.sh index 0c11f6a..5925037 100644 --- a/tests.sh +++ b/tests.sh @@ -1,6 +1,6 @@ #!/bin/sh -# $Id: tests.sh,v 1.114 2010/08/20 02:05:26 gilles Exp gilles $ +# $Id: tests.sh,v 1.116 2010/09/06 01:06:52 gilles Exp gilles $ # Example 1: # CMD_PERL='perl -I./Mail-IMAPClient-3.25/lib' sh -x tests.sh @@ -459,6 +459,18 @@ ll_maxage() --maxage 1 } +ll_newmessage() +{ + can_send && sendtestmessage + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --maxage 1 --folder INBOX --nofoldersizes +} + + ll_maxage_9999() { # can_send && sendtestmessage @@ -663,19 +675,6 @@ ll_bad_host_ssl() } -ll_justfoldersizes() -{ - $CMD_PERL ./imapsync \ - --host1 $HOST1 --user1 tata \ - --passfile1 ../../var/pass/secret.tata \ - --host2 $HOST2 --user2 titi \ - --passfile2 ../../var/pass/secret.titi \ - --justfoldersizes \ - --allow3xx -} - - - ll_useheader() { $CMD_PERL ./imapsync \ @@ -927,43 +926,38 @@ ll_authmech_CRAMMD5() { ll_delete2() { if can_send; then sendtestmessage titi - else - : fi - $CMD_PERL ./imapsync \ - --host1 $HOST1 --user1 tata \ - --passfile1 ../../var/pass/secret.tata \ - --host2 $HOST2 --user2 titi \ - --passfile2 ../../var/pass/secret.titi \ - --folder INBOX \ - --delete2 --expunge2 + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX \ + --delete2 --expunge2 } ll_delete() { if can_send; then sendtestmessage titi - else - : fi - $CMD_PERL ./imapsync \ - --host1 $HOST1 --user1 titi \ - --passfile1 ../../var/pass/secret.titi \ - --host2 $HOST2 --user2 tata \ - --passfile2 ../../var/pass/secret.tata \ - --folder INBOX \ - --delete --expunge + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 titi \ + --passfile1 ../../var/pass/secret.titi \ + --host2 $HOST2 --user2 tata \ + --passfile2 ../../var/pass/secret.tata \ + --folder INBOX \ + --delete --expunge } ll_bigmail() { - $CMD_PERL ./imapsync \ - --host1 $HOST1 --user1 tata \ - --passfile1 ../../var/pass/secret.tata \ - --host2 $HOST2 --user2 titi \ - --passfile2 ../../var/pass/secret.titi \ - --folder INBOX.bigmail \ - --allow3xx - echo 'rm /home/vmail/titi/.bigmail/cur/*' + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.bigmail + echo 'sudo rm -v /home/vmail/titi/.bigmail/cur/*' } @@ -1190,11 +1184,30 @@ dprof_bigfolder() } date2=`date` echo3 "[$date1] [$date2]" - mv tmon.out dprof_bigfolder_tmon.out + mv tmon.out dprof_bigfolder_tmon.out dprofpp -O 30 dprof_bigfolder_tmon.out dprofpp -O 30 -I dprof_bigfolder_tmon.out } +dprof_bigmail() +{ + date1=`date` + { $CMD_PERL -d:DProf ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.bigmail + echo 'sudo rm -v /home/vmail/titi/.bigmail/cur/*' || \ + true + } + date2=`date` + echo3 "[$date1] [$date2]" + mv tmon.out dprof_bigmail_tmon.out + dprofpp -O 30 dprof_bigmail_tmon.out + dprofpp -O 30 -I dprof_bigmail_tmon.out +} + @@ -1405,7 +1418,6 @@ ll_sep2 ll_bad_login ll_bad_host ll_bad_host_ssl -ll_justfoldersizes ll_useheader ll_regexmess ll_regexmess_scwchu @@ -1433,7 +1445,8 @@ gmail_gmail gmail_gmail2 archiveopteryx_1 allow3xx -noallow3xx' +noallow3xx +ll_newmessage' other_tests=' msw