From 0df034010bebc3fec42c8ea7dac688cd2a36a314 Mon Sep 17 00:00:00 2001 From: Nick Bebout Date: Tue, 27 Nov 2012 16:32:36 -0600 Subject: [PATCH] 1.516 --- CREDITS | 5 +- ChangeLog | 45 +- FAQ | 17 +- Makefile | 28 +- README | 8 +- TODO | 6 +- VERSION | 2 +- VERSION_EXE | 2 +- W/.BUILD_EXE_TIME | 4 + W/ml_announce.in | 4 +- W/paypal_reply/memo | 10 +- W/paypal_reply/paypal_bilan | 13 +- W/paypal_reply/paypal_bilan_1.60 | 1173 ++++++++++++++++++++++++++ W/paypal_reply/paypal_build_invoices | 11 +- W/test2.bat | 12 +- imapsync | 319 ++++--- index.shtml | 64 +- tests.sh | 133 ++- 18 files changed, 1673 insertions(+), 183 deletions(-) create mode 100755 W/paypal_reply/paypal_bilan_1.60 diff --git a/CREDITS b/CREDITS index 7b4671a..814817d 100644 --- a/CREDITS +++ b/CREDITS @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: CREDITS,v 1.170 2012/08/10 11:29:41 gilles Exp gilles $ +# $Id: CREDITS,v 1.171 2012/10/01 08:58:58 gilles Exp gilles $ If you want to make a donation to the author, Gilles LAMIRAL, use any of the following ways: @@ -30,6 +30,9 @@ I thank very much all of these people. I thank also very much all people who bought imapsync from the homepage but I don't cite them here. +Jeff Verheyen +Contributed by giving an 32GB microSDHC card and the book +29.69 "Design by Nature: Using Universal Forms and Principles in Design" David Abrahams Gave a patch for --authmech PREAUTH diff --git a/ChangeLog b/ChangeLog index 4ddd00e..c833a49 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,17 +1,54 @@ RCS file: RCS/imapsync,v Working file: imapsync -head: 1.508 +head: 1.516 branch: locks: strict - gilles: 1.508 + gilles: 1.516 access list: symbolic names: keyword substitution: kv -total revisions: 508; selected revisions: 508 +total revisions: 516; selected revisions: 516 description: ---------------------------- -revision 1.508 locked by: gilles; +revision 1.516 locked by: gilles; +date: 2012/11/02 22:15:04; author: gilles; state: Exp; lines: +43 -38 +Added current date at the beginning of the run, useful when imapsync doesn't end properly or hasn't finished yet. +Better output for diff statistics. +---------------------------- +revision 1.515 +date: 2012/10/31 12:56:02; author: gilles; state: Exp; lines: +12 -9 +Added Initial difference to compare it with Final difference. +---------------------------- +revision 1.514 +date: 2012/10/31 01:39:38; author: gilles; state: Exp; lines: +32 -16 +Added --foldersizesatend and --nofoldersizesatend, on by default. +Added statistic host2 minus host1: number of messages and bytes. +---------------------------- +revision 1.513 +date: 2012/10/30 22:31:49; author: gilles; state: Exp; lines: +40 -19 +Add --fixslash2 to avoid 'Invalid mailbox name' when --sep2 is not / and sep1 is / and host1 folders contain --sep2 characters. +---------------------------- +revision 1.512 +date: 2012/10/30 01:44:26; author: gilles; state: Exp; lines: +22 -8 +Added option --showpasswords +--showpasswords : shows passwords on output instead of "MASKED". +Off by default. +---------------------------- +revision 1.511 +date: 2012/10/27 22:37:57; author: gilles; state: Exp; lines: +44 -20 +Added --delete2duplicates; Delete messages in host2 that are duplicates. +--delete2duplicates is on when --delete2 is on unless --nodelete2duplicates. +---------------------------- +revision 1.510 +date: 2012/10/27 15:18:44; author: gilles; state: Exp; lines: +36 -30 +Added --pidfilelocking option to abort in case another imapsync may be running. +---------------------------- +revision 1.509 +date: 2012/10/24 14:57:56; author: gilles; state: Exp; lines: +41 -13 +Added option --noabletosearch to allow listing messages without SEARCH command. Hack for imap server softalk 7.6.4. (8.6 is fine about search) +---------------------------- +revision 1.508 date: 2012/09/10 21:10:13; author: gilles; state: Exp; lines: +81 -17 Added ETA after each copy. Estimated Time of Arrival. ---------------------------- diff --git a/FAQ b/FAQ index 1f12251..3f4e725 100644 --- a/FAQ +++ b/FAQ @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: FAQ,v 1.116 2012/09/11 21:00:06 gilles Exp gilles $ +# $Id: FAQ,v 1.117 2012/11/03 00:44:30 gilles Exp gilles $ +------------------+ | FAQ for imapsync | @@ -1203,10 +1203,15 @@ imapsync --host1 mail.oldhost.com \ --user2 my_email@gmail.com \ --password2 password \ --folder "INBOX.Sent" \ - --regextrans2 "s/Sent/Sent Mail/" + --regextrans2 "s/Sent/Sent Mail/" \ + --exitwhenover 500000000 The same goes for the "All Mail" archive pseudo-folder. +--exitwhenover option is here to avoid locking when transfers +exceed maximum limit. +See http://support.google.com/a/bin/answer.py?hl=en&answer=1071518 + ======================================================================= Q. Synchronizing from Gmail to XXX @@ -1218,6 +1223,7 @@ R. Gmail needs SSL --authmech1 LOGIN \ --user1 gilles.lamiral@gmail.com \ --password1 gmailsecret \ + --exitwhenover 2500000000 \ --host2 localhost --user2 tata \ --password2 tatasecret \ @@ -1229,7 +1235,12 @@ option: --regextrans2 's/\[Gmail\]/Gmail/' You can select folders exported to imap within the gmail preferences, -unselect all "System labels" +unselect all "System labels". + +--exitwhenover option is here to avoid locking when transfers +exceed maximum limit. +See http://support.google.com/a/bin/answer.py?hl=en&answer=1071518 + ======================================================================= Q. migrate email from gmail to google apps diff --git a/Makefile b/Makefile index 02ae603..fe1d476 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -# $Id: Makefile,v 1.107 2012/09/11 21:00:06 gilles Exp gilles $ +# $Id: Makefile,v 1.110 2012/11/03 01:38:12 gilles Exp gilles $ .PHONY: help usage all @@ -108,14 +108,16 @@ test_quick_229: imapsync tests.sh test_quick_3xx: imapsync tests.sh CMD_PERL='perl -I./$(IMAPClient_3xx)' /usr/bin/time sh -x tests.sh locallocal -testv2: .test_229 - CMD_PERL='perl -I./$(IMAPClient_2xx) /usr/bin/time sh tests.sh +testv2: + CMD_PERL='perl -I./$(IMAPClient_2xx)' /usr/bin/time sh tests.sh touch .test_229 -testv3:.test_3xx - CMD_PERL='perl -I./$(IMAPClient_3xx)' sh -x tests.sh +testv3: + CMD_PERL='perl -I./$(IMAPClient_3xx)' /usr/bin/time sh tests.sh touch .test_3xx +testv: testv2 testv3 + test: .test_229 .test_3xx tests: test @@ -210,7 +212,6 @@ imapsync_elf_x86.bin: imapsync lfo: cidone niouze_lfo upload_lfo -dist: cidone test clean all INSTALL dist_prepa dist_prepa_exe tarball: cidone all echo making tarball $(DIST_FILE) @@ -232,6 +233,9 @@ DIST_PATH := ./dist/$(DIST_SECRET) lalala: echo $(DIST_SECRET) +dist: cidone test clean all INSTALL dist_prepa dist_prepa_exe + + dist_prepa: tarball dist_dir ln -f ../prepa_dist/$(DIST_FILE) $(DIST_PATH)/ rcsdiff imapsync @@ -261,10 +265,10 @@ ks: . imapsync@ks.lamiral.info:public_html/imapsync/ ksa: - rsync -avHz --delete \ + rsync -avHz --delete -P \ . imapsync@ks.lamiral.info:public_html/imapsync/ -publish: upload_ks ks ml +publish: upload_ks ksa ml PUBLIC_FILES = ./ChangeLog ./COPYING ./CREDITS ./FAQ \ ./index.shtml ./INSTALL \ @@ -283,7 +287,7 @@ ml: mailq -upload_ks: ci +upload_ks: ci dist rsync -lptvHzP $(PUBLIC_FILES) \ root@ks.lamiral.info:/var/www/imapsync/ rsync -lptvHzP $(PUBLIC_FILES_W) \ @@ -307,10 +311,10 @@ upload_lfo: /home/gilles/public_html/www.linux-france.org/html/prj/imapsync/.htaccess sh ~/memo/lfo-rsync -upload_index: index.shtml FAQ +upload_index: index.shtml FAQ COPYING CREDITS validate --verbose index.shtml - rcsdiff index.shtml FAQ COPYING - rsync -avH index.shtml FAQ COPYING root@ks.lamiral.info:/var/www/imapsync/ + rcsdiff index.shtml FAQ COPYING CREDITS + rsync -avH index.shtml FAQ COPYING CREDITS root@ks.lamiral.info:/var/www/imapsync/ niouze_lfo : echo "CORRECT ME: . ./memo && lfo_announce" diff --git a/README b/README index bb25d43..19bea6b 100644 --- a/README +++ b/README @@ -3,7 +3,7 @@ NAME Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 44 different IMAP server softwares supported with success. - $Revision: 1.508 $ + $Revision: 1.516 $ SYNOPSIS To synchronise imap account "foo" on "imap.truc.org" to imap account @@ -85,7 +85,7 @@ USAGE [--expunge] [--expunge1] [--expunge2] [--uidexpunge2] [--delete2folders] [--delete2foldersonly] [--delete2foldersbutnot] [--subscribed] [--subscribe] [--subscribe_all] - [--nofoldersizes] + [--nofoldersizes] [--nofoldersizesatend] [--dry] [--debug] [--debugimap][--debugimap1][--debugimap2] [--timeout ] [--fast] @@ -354,7 +354,7 @@ IMAP SERVERS - Qualcomm Worldmail (NT) - Rockliffe Mailsite 5.3.11, 4.5.6 - Samsung Contact IMAP server 8.5.0 - - Scalix v10.1, 10.0.1.3, 11.0.0.431 + - Scalix v10.1, 10.0.1.3, 11.0.0.431, 11.4.6 - SmarterMail, Smarter Mail 5.0 Enterprise, Smarter Mail 5.5 [host1]. - SunONE Messaging server 5.2, 6.0 (SUN JES - Java Enterprise System) - Sun Java(tm) System Messaging Server 6.2-2.05, 6.2-7.05, 6.3 @@ -440,5 +440,5 @@ SIMILAR SOFTWARES Feedback (good or bad) will often be welcome. - $Id: imapsync,v 1.508 2012/09/10 21:10:13 gilles Exp gilles $ + $Id: imapsync,v 1.516 2012/11/02 22:15:04 gilles Exp gilles $ diff --git a/TODO b/TODO index 8549e45..85b29a3 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: TODO,v 1.113 2012/09/11 20:58:32 gilles Exp gilles $ +# $Id: TODO,v 1.114 2012/11/02 22:19:14 gilles Exp gilles $ TODO file for imapsync ---------------------- @@ -62,7 +62,6 @@ http://mail.google.com/support/bin/answer.py?answer=78892 Look at larch https://github.com/rgrove/larch/ http://wonko.com/ - Add and option to sync to & from files. Add an --aclregextrans2 flag. @@ -94,7 +93,6 @@ Fix bug "not possible to use space in the imap password" Add kerberos authentification - Add stdin/stdout filter before transfer: "Now i asked me, how to modify your perl program to work with that - in example, to write each mail to stdout, pipe that to the @@ -142,6 +140,8 @@ http://asg.web.cmu.edu/cyrus/download/imapd/altnamespace.html =========================================================================== +DONE. Add current date at the beginning of the run, useful when imapsync +doesn't finish with statistics. DONE. Not donse since useless now (--useuid) Add a --skipheaderinfolder option diff --git a/VERSION b/VERSION index 5047ace..c9cb815 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.508 +1.516 diff --git a/VERSION_EXE b/VERSION_EXE index 5047ace..c9cb815 100644 --- a/VERSION_EXE +++ b/VERSION_EXE @@ -1 +1 @@ -1.508 +1.516 diff --git a/W/.BUILD_EXE_TIME b/W/.BUILD_EXE_TIME index b1560eb..c2fec3c 100644 --- a/W/.BUILD_EXE_TIME +++ b/W/.BUILD_EXE_TIME @@ -156,3 +156,7 @@ 1347196697 END 1.507 : dimanche 9 septembre 2012, 15:18:17 (UTC+0200) 1347392501 BEGIN 1.508 : mardi 11 septembre 2012, 21:41:41 (UTC+0200) 1347393591 END 1.508 : mardi 11 septembre 2012, 21:59:51 (UTC+0200) +1351688288 BEGIN 1.515 : mercredi 31 octobre 2012, 13:58:08 (UTC+0100) +1351689817 END 1.515 : mercredi 31 octobre 2012, 14:23:37 (UTC+0100) +1351894814 BEGIN 1.516 : vendredi 2 novembre 2012, 23:20:14 (UTC+0100) +1351896296 END 1.516 : vendredi 2 novembre 2012, 23:44:56 (UTC+0100) diff --git a/W/ml_announce.in b/W/ml_announce.in index 50666f6..6392d6f 100644 --- a/W/ml_announce.in +++ b/W/ml_announce.in @@ -1,4 +1,4 @@ -m4_dnl $Id: ml_announce.in,v 1.3 2012/09/11 21:01:53 gilles Exp gilles $ +m4_dnl $Id: ml_announce.in,v 1.4 2012/11/03 01:38:45 gilles Exp gilles $ m4_dnl m4_define(`M4_imapsync_VERSION',m4_esyscmd(cat VERSION|tr -d '\n'))m4_dnl m4_define(`M4_SECRET_PATH',m4_esyscmd(cat dist/path_last.txt|tr -d '\n'))m4_dnl @@ -10,6 +10,8 @@ To: imapsync_update@lists.lamiral.info Hello imapsync user, +Sorry for the previous link given, it was the one of release 1.508 + You're subscribed to the newsletter announcing imapsync new releases (very few traffic) and the way to get them. Send me a note if you don't want to receive those announces anymore. diff --git a/W/paypal_reply/memo b/W/paypal_reply/memo index 87232d5..c62e4e8 100644 --- a/W/paypal_reply/memo +++ b/W/paypal_reply/memo @@ -1,6 +1,6 @@ #!/bin/sh -# $Id: memo,v 1.11 2012/08/16 13:37:02 gilles Exp gilles $ +# $Id: memo,v 1.12 2012/11/02 07:49:37 gilles Exp gilles $ echo paypal_bilan_todo @@ -28,12 +28,12 @@ Europe a un autre assujetti : Article 262 ter => Exoneration EOF } -echo paypal_bilan_exportbnc -paypal_bilan_exportbnc() { -# DID output no diff between paypal_bilan_1.58 and 1.59 +echo paypal_bilan_Avant_commission +paypal_bilan_Avant_commission() { +# DID output diff between paypal_bilan_1.60 and 1.61 ( #set -x -/g/public_html/imapsync/W/paypal_reply/paypal_bilan_1.58 --bnc --debug --debug_invoice --first_in 147 \ +/g/public_html/imapsync/W/paypal_reply/paypal_bilan_1.60 --bnc --debug --debug_invoice --first_in 147 \ --avoid_numbers '292 293 643 644 731 732 1093 1330 1331 1332 1333 1334 1652 1653' \ /g/paypal/paypal_201?_??_complet.csv \ > /g/var/paypal_bilan/tests/paypal_invoice.out1 2>&1 diff --git a/W/paypal_reply/paypal_bilan b/W/paypal_reply/paypal_bilan index 10b7a09..4cc355e 100755 --- a/W/paypal_reply/paypal_bilan +++ b/W/paypal_reply/paypal_bilan @@ -1,6 +1,6 @@ #!/usr/bin/perl -# $Id: paypal_bilan,v 1.58 2012/08/11 00:01:46 gilles Exp gilles $ +# $Id: paypal_bilan,v 1.61 2012/11/02 01:31:58 gilles Exp gilles $ use strict; use warnings; @@ -13,7 +13,7 @@ use Test::More 'no_plan' ; die unless (utf8_supported_charset('ISO-8859-1')); -my $rcs = '$Id: paypal_bilan,v 1.58 2012/08/11 00:01:46 gilles Exp gilles $ ' ; +my $rcs = '$Id: paypal_bilan,v 1.61 2012/11/02 01:31:58 gilles Exp gilles $ ' ; $rcs =~ m/,v (\d+\.\d+)/ ; my $VERSION = ($1) ? $1: "UNKNOWN" ; @@ -111,7 +111,8 @@ foreach my $file ( @files ) { 'Devise', 'Montant', "Numéro d'avis de réception", 'Solde', 'Pays', 'Nom Option 1', 'Valeur Option 1', 'Hors taxe', "Titre de l'objet", 'Nom Option 2', 'Option 2 Valeur') } ; #print "$Nom\n" ; - ( $Etat ) = @action{ ( 'Etat', ) } || @action{ ( 'État' ) } ; + ( $Etat ) = @action{ ( 'Etat' ) } || @action{ ( 'État' ) } ; + ( $Hors_taxe ) = @action{ ( 'Hors taxe' ) } || @action{ ( 'Avant commission' ) } ; my $invoice = 'NONE' ; $Montant = $action->{ Net } if not defined $Montant; compute_line($action, $invoice, $Date, $Heure, $Fuseau_horaire, $Nom, $Type, $Etat, @@ -200,6 +201,7 @@ print "Nb invoice sent $nb_invoice_sent\n" ; print "Have to send invoices @invoice_not_sent\n" if ( @invoice_not_sent ) ; my $total_eur2 = $total_HT_EUR_exo + $total_HT_EUR_ass + $total_TVA_EUR + $total_HT_EUR_sup + $total_TVA_EUR_sup ; +$total_eur2 = sprintf('%2.2f', $total_eur2) ; print "$total_eur != $total_eur2 = $total_HT_EUR_exo + $total_HT_EUR_ass + $total_TVA_EUR + $total_HT_EUR_sup + $total_TVA_EUR_sup\n" if ( $total_eur != $total_eur2 ) ; @@ -523,13 +525,10 @@ sub build_invoice { 'Etat/Province/Région/Comté/Territoire/Préfecture/République', 'Code postal', 'Pays', 'line_number', 'line_csv', 'file_csv', 'Nom Option 2', 'Option 2 Valeur' ) } ; - my( $Etat_Province1 ) = @action{ ( 'Etat/Province/Région/Comté/Territoire/Préfecture/République' ) } ; - my( $Etat_Province2 ) = @action{ ( 'État/Province/Région/Comté/Territoire/Préfecture/République' ) } ; - $Etat_Province = @action{ ( 'Etat/Province/Région/Comté/Territoire/Préfecture/République' ) } || @action{ ( 'État/Province/Région/Comté/Territoire/Préfecture/République' ) } || '' ; - + ( $Hors_taxe ) = @action{ ( 'Hors taxe' ) } || @action{ ( 'Avant commission' ) } ; #print "$Hors_taxe $Devise\n" ; my $Hors_taxe_num = $Hors_taxe ; $Hors_taxe_num =~ s{,}{.} ; diff --git a/W/paypal_reply/paypal_bilan_1.60 b/W/paypal_reply/paypal_bilan_1.60 new file mode 100755 index 0000000..62b9a03 --- /dev/null +++ b/W/paypal_reply/paypal_bilan_1.60 @@ -0,0 +1,1173 @@ +#!/usr/bin/perl + +# $Id: paypal_bilan,v 1.60 2012/11/02 01:15:48 gilles Exp gilles $ + +use strict; +use warnings; +use Getopt::Long; +use Text::CSV_XS ; +use IO::Handle ; +use Data::Dumper ; +use Unicode::MapUTF8 qw(to_utf8 from_utf8 utf8_supported_charset); +use Test::More 'no_plan' ; + +die unless (utf8_supported_charset('ISO-8859-1')); + +my $rcs = '$Id: paypal_bilan,v 1.60 2012/11/02 01:15:48 gilles Exp gilles $ ' ; +$rcs =~ m/,v (\d+\.\d+)/ ; +my $VERSION = ($1) ? $1: "UNKNOWN" ; + + +my $total_usd_received = 0 ; +my $total_usd_invoice = 0 ; +my $total_HT_EUR_exo = 0 ; +my $total_HT_EUR_ass = 0 ; +my $total_TVA_EUR = 0 ; + +my $total_HT_EUR_sup = 0 ; +my $total_TVA_EUR_sup = 0 ; + +my $total_eur_received = 0 ; +my $total_eur_invoice = 0 ; +my $nb_invoice = 0 ; +my $nb_invoice_refund = 0 ; +my $nb_invoice_suspended = 0 ; +my $nb_invoice_canceled = 0 ; + +my ( $tests, $testeur ) ; +my $debug ; +my $debug_csv ; +my $debug_dev ; +my $debug_invoice ; +my $debug_invoice_utf8 ; + +my $first_invoice = 1 ; +my $print_details = '' ; +my $bnc = '' ; +my $exportbnc = '' ; + +my $usdeur = 1.2981 ; +my $invoices ; +my %invoice_refund ; +my %invoice_canceled ; +my %invoice_suspended ; +my $write_invoices = 0 ; +my $avoid_numbers ; + +my $dir_invoices = '/g/var/paypal_invoices' ; + +my $option_ret = GetOptions ( + 'tests' => \$tests, + 'debug' => \$debug, + 'debug_csv' => \$debug_csv, + 'debug_dev' => \$debug_dev, + 'debug_invoice' => \$debug_invoice, + 'debug_invoice_utf8' => \$debug_invoice_utf8, + + 'first_invoice=i' => \$first_invoice, + 'print_details|details' => \$print_details, + 'bnc' => \$bnc, + 'exportbnc=s' => \$exportbnc, + 'usdeur=f' => \$usdeur, + 'invoices=s' => \$invoices, + 'write_invoices!' => \$write_invoices, + 'avoid_numbers=s' => \$avoid_numbers, +); + +$testeur = Test::More->builder ; +$testeur->no_ending(1) ; + +if ( $tests ) { + $testeur->no_ending( 0 ) ; + exit( tests( ) ) ; +} + + +my @files = @ARGV ; +my %action_of_invoice ; + +my %invoice_paypal ; +#$invoice_paypal{ $first_invoice } = 1 ; + +my @invoices_wanted = split( /\s+/, $invoices ) if $invoices ; + +my @avoid_numbers = split( /\s+/, $avoid_numbers ) if $avoid_numbers ; +my %avoid_numbers ; +@avoid_numbers{ @avoid_numbers } = ( ) if @avoid_numbers ; + +#print "@invoices\n" ; + +foreach my $file ( @files ) { + + my @actions = parse_file( $file ) ; + + foreach my $action (@actions) { + my %action = %$action ; + #print $action->{ Nom }, "\n" ; + my( $Date, $Heure, $Fuseau_horaire, $Nom, $Type, $Etat, + $Devise, $Montant, $Numero_davis_de_reception, $Solde, + $Pays, $Nom_Option_1, $Valeur_Option_1, $Hors_taxe, $Titre_de_l_objet, $Nom_Option_2, $Option_2_Valeur ) + = @action{ ( 'Date', 'Heure', 'Fuseau horaire', 'Nom', 'Type', 'Etat', + 'Devise', 'Montant', "Numéro d'avis de réception", 'Solde', + 'Pays', 'Nom Option 1', 'Valeur Option 1', 'Hors taxe', "Titre de l'objet", 'Nom Option 2', 'Option 2 Valeur') } ; + #print "$Nom\n" ; + ( $Etat ) = @action{ ( 'Etat', ) } || @action{ ( 'État' ) } ; + my $invoice = 'NONE' ; + $Montant = $action->{ Net } if not defined $Montant; + compute_line($action, $invoice, $Date, $Heure, $Fuseau_horaire, $Nom, $Type, $Etat, + $Devise, $Montant, $Numero_davis_de_reception, $Solde, + $Pays, $Nom_Option_1, $Valeur_Option_1, $Hors_taxe, $Titre_de_l_objet ) ; + + # index by invoice number + $action_of_invoice{ $action->{ 'invoice' } } = $action ; + } + delete $action_of_invoice{ 'NONE' } ; +} + +my $last_invoice ; +my @invoice_paypal = sort { $a <=> $b } keys %invoice_paypal ; +$last_invoice = $invoice_paypal[-1] || 0 ; +my $first_invoice_paypal = $invoice_paypal[0] || 0 ; + +@invoices_wanted = ( $first_invoice .. $last_invoice ) if ( ! @invoices_wanted ) ; + +my @invoice_sent ; +my %invoice_sent ; +my @invoice_not_sent ; +my %invoice_not_sent ; + +foreach my $invoice ( @invoices_wanted ) { + + my $action = $action_of_invoice{ $invoice } ; + next if ! $action ; + my $email_address = $action->{ "De l'adresse email" } ; + + my $invoice_sent = invoice_sent( $dir_invoices, $invoice, $email_address ) ; + #print "$invoice $invoice_sent\n" ; + + if ( $invoice_sent ) { + $invoice_sent{ $invoice }++ ; + #build_invoice( $invoice ) ; + }elsif( not ( $invoice_canceled{ $invoice } or $invoice_refund{ $invoice } ) ) { + $invoice_not_sent{ $invoice }++ ; + build_invoice( $invoice ) ; + } +} + +@invoice_sent = sort { $a <=> $b } keys( %invoice_sent ) ; +my $nb_invoice_sent = scalar( @invoice_sent ) ; +@invoice_not_sent = sort { $a <=> $b } keys( %invoice_not_sent ) ; + +my @invoice_canceled = sort { $a <=> $b } keys( %invoice_canceled ) ; +my @invoice_suspended = sort { $a <=> $b } keys( %invoice_suspended ) ; +my @invoice_refund = sort { $a <=> $b } keys( %invoice_refund ) ; + + +print( "\n", "=" x 60, "\n" ) if $bnc ; + + + +print "USD banque $total_usd_received\n" ; +print "USD invoice $total_usd_invoice\n" ; +my $total_eur_from_usd ; +$total_eur_from_usd = sprintf('%2.2f', $total_usd_invoice / $usdeur ) ; # au 30 nov 2010 http://fr.finance.yahoo.com/devises/convertisseur/#from=EUR;to=USD;amt=1 +print "EUR from USD $total_eur_from_usd\n" ; +print "EUR banque $total_eur_received\n" ; +print "EUR invoice $total_eur_invoice\n" ; + +my $total_eur = $total_eur_from_usd + $total_eur_invoice ; + +$total_HT_EUR_exo = sprintf('%2.2f', $total_HT_EUR_exo) ; +$total_HT_EUR_ass = sprintf('%2.2f', $total_HT_EUR_ass) ; +$total_TVA_EUR = sprintf('%2.2f', $total_TVA_EUR) ; + +$total_HT_EUR_sup = sprintf('%2.2f', $total_HT_EUR_sup) ; +$total_TVA_EUR_sup = sprintf('%2.2f', $total_TVA_EUR_sup) ; + +$total_eur = sprintf('%2.2f', $total_eur) ; + +print "EUR total $total_eur\n" ; +print "EUR total HT exo $total_HT_EUR_exo\n" ; +print "EUR total HT assuj $total_HT_EUR_ass\n" ; +print "EUR total TVA $total_TVA_EUR\n" ; +print "EUR total HT sup $total_HT_EUR_sup\n" ; +print "EUR total TVA sup $total_TVA_EUR_sup\n" ; +print "Nb invoice $nb_invoice ( from $first_invoice_paypal to $last_invoice )\n" ; +print "Nb invoice canceled ($nb_invoice_canceled) @invoice_canceled\n" ; +print "Nb invoice suspended ($nb_invoice_suspended) @invoice_suspended\n" ; +print "Nb invoice refund ($nb_invoice_refund) @invoice_refund\n" ; +print "Nb invoice sent $nb_invoice_sent\n" ; +print "Have to send invoices @invoice_not_sent\n" if ( @invoice_not_sent ) ; + +my $total_eur2 = $total_HT_EUR_exo + $total_HT_EUR_ass + $total_TVA_EUR + $total_HT_EUR_sup + $total_TVA_EUR_sup ; +$total_eur2 = sprintf('%2.2f', $total_eur2) ; +print "$total_eur != $total_eur2 = $total_HT_EUR_exo + $total_HT_EUR_ass + $total_TVA_EUR + $total_HT_EUR_sup + $total_TVA_EUR_sup\n" +if ( $total_eur != $total_eur2 ) ; + +sub parse_one_line_io { + my $csv = shift ; + my $io = shift ; + + my $line = $csv->getline($io) ; + + return if ( $csv->eof( ) ) ; + if ( not defined( $line ) ) { + my($cde, $str, $pos) = $csv->error_diag () ; + print "[$cde] [$str] [$pos]\n" ; + + } + return( $line ) ; +} + +sub hash_and_count_dupplicate { + my @columns = @_ ; + my %columns ; + + #@columns_def{ @columns_def } = ( ) ; + foreach my $col ( @columns ) { + $columns{ $col } += 1 ; + } + $debug_csv and print "Nb columns: ", scalar( keys %columns ), " ", scalar( @columns ), "\n" ; + # debug how many time a title is defined + foreach my $col (1 .. scalar( @columns )) { + $debug_csv and print "$col | ", + deci_to_AA( $col ) , " | ", + $columns{ $columns[ $col - 1 ] }, " | ", + $columns[ $col - 1 ], "\n" ; + } + + # exit in case two columns have the same name + die "Erreur : doublons dans les titres\n" if ( scalar( keys %columns ) != scalar( @columns ) ) ; + + return( %columns ) ; +} + +sub deci_to_AA { + my $deci = shift ; + my $AA = ''; + + while ( $deci > 0 ) { + my $quot = int( ( $deci - 1 ) / 26 ) ; + my $rest = $deci - 1 - ( 26 * $quot ) ; + my $char = chr ( ord('A') + $rest ) ; + $AA = $char . $AA ; + $deci = $quot ; + } + #print "col=$AA\n" ; + return( $AA ) ; +} + +sub remove_first_blank { + my $string = shift ; + + $string =~ s/^ +// ; + return( $string ) ; + +} + +sub parse_file { + my $file = shift ; + + open my $io, "<", $file or die "$file: $!" ; + + my $csv = Text::CSV_XS->new( { + sep_char => ',', + binary => 1, + keep_meta_info => 1, + eol => $/, + } ) ; + + my $line_1 = parse_one_line_io( $csv, $io ) ; + die if ( not defined $line_1 ) ; # first line must have no problem + + my @columns_def_orig = @$line_1 ; + my @columns_def = map { remove_first_blank( $_ ) } @columns_def_orig ; + $debug_csv and print "columns_def = ", map( { "[$_]" } @columns_def ), "\n"; + + my %columns_def = hash_and_count_dupplicate( @columns_def ) ; + my $nb_columns_def = scalar @columns_def ; + + my $line_counter = 2 ; + my @actions ; + while ( 1 ) { + $debug_csv and print "ligne $line_counter ", $csv->eof( ), "\n" ; + my $line = parse_one_line_io( $csv, $io ) ; + last if ( $csv->eof( ) ) ; + if ( not defined $line ) { + print "Erreur ligne $line_counter : ", $csv->error_diag, "\n\n"; + ++$line_counter ; + next ; + } + my @columns = @$line ; + + if ( $nb_columns_def != scalar @columns ) { + print "Erreur ligne $line_counter : nombre de colonnes = ", scalar @columns, " != $nb_columns_def\n" ; + ++$line_counter ; + next ; + } + my %columns ; + @columns{ @columns_def } = @columns ; + $columns{ 'file_csv' } = $file ; + $columns{ 'line_number' } = $line_counter ; + $csv->combine( @columns ) ; + my $line_csv = $csv->string(); + $columns{ 'line_csv' } = $line_csv ; + $debug_csv and print map( { "[$_] = [" . $columns{$_} . "]\n" } + @columns_def, 'line_number', 'line_csv', 'file_csv' ), + "\n"; + ++$line_counter ; + push( @actions, \%columns ) ; + } + close( $io ); + return( reverse @actions ) ; +} + +sub next_invoice { + my @current_numbers = sort { $a <=> $b } ( $first_invoice - 1, keys( %invoice_paypal ) ) ; + my $last_invoice = $current_numbers[ -1 ] || 0 ; + + #keys( %avoid_numbers ), + my $next_invoice = $last_invoice + 1 ; + while ( exists( $avoid_numbers{ $next_invoice } ) ) { $next_invoice++ ; } + $invoice_paypal{ $next_invoice } = 1 ; + #print "AAA [@current_numbers] [$last_invoice] [$next_invoice]\n" ; + + return( $next_invoice ) ; +} + +sub keyval { + my %hash = @_ ; + return( join( " ", map( { "$_ => " . $hash{ $_ } } keys %hash ) ) . "\n" ) ; +} + + +sub tests_next_invoice { + ok( 1 == next_invoice( ), 'next_invoice: 1' ) ; + ok( 2 == next_invoice( ), 'next_invoice: 2' ) ; + @avoid_numbers{ (3, 4, 6, 8 ) } = ( ) ; + ok( 5 == next_invoice( ), 'next_invoice: 7' ) ; + ok( 7 == next_invoice( ), 'next_invoice: 8' ) ; + ok( 9 == next_invoice( ), 'next_invoice: 9' ) ; + %invoice_paypal = () ; + $first_invoice = 7 ; + ok( 7 == next_invoice( ), 'next_invoice: 7' ) ; +} + + +sub tests { + tests_next_invoice( ) ; + #tests_half( ) ; + tests_cut( ) ; +} + +sub compute_line { + my( $action, $invoice, $Date, $Heure, $Fuseau_horaire, $Nom, $Type, $Etat, + $Devise, $Montant, $Numero_davis_de_reception, $Solde, + $Pays, $Nom_Option_1, $Valeur_Option_1, $Hors_taxe_paypal, $Titre_de_l_objet ) = @_ ; + + $debug and print( "-" x 60, "\n", + "[$Date] [$Heure] [$Fuseau_horaire] [$Nom] [$Type] [$Etat] ", + "[$Devise] [$Hors_taxe_paypal] [$Montant] [$Numero_davis_de_reception] [$Solde]\n", + "[$Pays] [$Nom_Option_1] [$Valeur_Option_1] [$Titre_de_l_objet]\n" ) ; + + $Montant =~ s/[^0-9-,.]//g ; + $Montant =~ s/,/./g ; + #$debug and print "MM[$Montant]\n" ; + $Hors_taxe_paypal =~ s/,/./g ; + + my $MontantEUR; + my( $montant_HT_EUR_exo, $montant_HT_EUR_ass, $montant_TVA_EUR ) ; + my( $montant_HT_EUR_sup, $montant_TVA_EUR_sup ) ; + + if ( $bnc ) { + $MontantEUR = $Montant ; + $MontantEUR = sprintf( "%.4f", $Montant/$usdeur ) if ($Devise eq 'USD') ; + print( "\n", "=" x 60, "\n" ) ; + print( "[$Date] [$Nom] [$Type] [$Etat] [$Devise] [$Hors_taxe_paypal] [$Montant] [EUR $MontantEUR]\n", + "[$Pays] [$Nom_Option_1] [$Valeur_Option_1] [$Titre_de_l_objet]\n" ) ; + } + + if ( + 'Paiement sur site marchand reçu' eq $Type + and 'USD' eq $Devise + and ( 'Terminé' eq $Etat or 'Compensé' eq $Etat ) + ) { + $Montant =~tr/,/./; + #print "$Montant\n" ; + my $Montant2_usd; + $Montant2_usd = $Hors_taxe_paypal ; + $total_usd_received += $Montant ; + $total_usd_invoice += $Montant2_usd ; + ( $montant_HT_EUR_exo, $montant_HT_EUR_ass, $montant_TVA_EUR, $montant_HT_EUR_sup, $montant_TVA_EUR_sup ) + = tva_line( $Devise, $Montant2_usd, $Pays, $Nom_Option_1, $Valeur_Option_1, $Titre_de_l_objet ) ; + $total_HT_EUR_exo += $montant_HT_EUR_exo ; + $total_HT_EUR_ass += $montant_HT_EUR_ass ; + $total_TVA_EUR += $montant_TVA_EUR ; + #$invoice = $first_invoice + $nb_invoice ; + $invoice = next_invoice( ) ; + $nb_invoice++ ; + $print_details and print ( "[$invoice] [$Date] [$Heure] [$Fuseau_horaire] [$Nom] [$Type] [$Etat] [$Devise] [$Montant] [$Numero_davis_de_reception] [$Solde]\n" ) ; + + } + + if ( + 'Paiement sur site marchand reçu' eq $Type + and 'EUR' eq $Devise + and ( 'Terminé' eq $Etat or 'Compensé' eq $Etat ) + ) { + $Montant =~tr/,/./; + #print "$Montant\n" ; + my $Montant2_eur; + $Montant2_eur = $Hors_taxe_paypal ; + $total_eur_received += $Montant ; + $total_eur_invoice += $Montant2_eur ; + ( $montant_HT_EUR_exo, $montant_HT_EUR_ass, $montant_TVA_EUR, $montant_HT_EUR_sup, $montant_TVA_EUR_sup ) + = tva_line( $Devise, $Montant2_eur, $Pays, $Nom_Option_1, $Valeur_Option_1, $Titre_de_l_objet ) ; + $total_HT_EUR_exo += $montant_HT_EUR_exo ; + $total_HT_EUR_ass += $montant_HT_EUR_ass ; + $total_TVA_EUR += $montant_TVA_EUR ; + $total_HT_EUR_sup += $montant_HT_EUR_sup ; + $total_TVA_EUR_sup += $montant_TVA_EUR_sup ; + + + #$invoice = $first_invoice + $nb_invoice ; + $invoice = next_invoice( ) ; + $nb_invoice++ ; + $print_details and print ( "[$invoice] [$Date] [$Heure] [$Fuseau_horaire] [$Nom] [$Type] [$Etat] [$Devise] [$Montant] [$Numero_davis_de_reception] [$Solde]\n" ) ; + } + + if ( + 'Paiement sur site marchand reçu' eq $Type + and 'EUR' eq $Devise + and 'Remboursé' eq $Etat + ) { + $invoice = next_invoice( ) ; + $nb_invoice++ ; + $nb_invoice_refund++; + $invoice_refund{ $invoice }++ ; + + $print_details and print ( "[$invoice] [$Date] [$Heure] [$Fuseau_horaire] [$Nom] [$Type] [$Etat] [$Devise] [$Montant] [$Numero_davis_de_reception] [$Solde]\n" ) ; + } + + if ( + 'Paiement sur site marchand reçu' eq $Type + and 'EUR' eq $Devise + and 'Annulé' eq $Etat + ) { + $invoice = next_invoice( ) ; + $nb_invoice++ ; + $nb_invoice_canceled++; + $invoice_canceled{ $invoice }++ ; + + $print_details and print ( "[$invoice] [$Date] [$Heure] [$Fuseau_horaire] [$Nom] [$Type] [$Etat] [$Devise] [$Montant] [$Numero_davis_de_reception] [$Solde]\n" ) ; + } + + if ( + 'Paiement sur site marchand reçu' eq $Type + and 'EUR' eq $Devise + and 'Suspendu' eq $Etat + ) { + $invoice = next_invoice( ) ; + $nb_invoice++ ; + $nb_invoice_suspended++; + $invoice_suspended{ $invoice }++ ; + + $print_details and print ( "[$invoice] [$Date] [$Heure] [$Fuseau_horaire] [$Nom] [$Type] [$Etat] [$Devise] [$Montant] [$Numero_davis_de_reception] [$Solde]\n" ) ; + } + + if ( + 'Paiement sur site marchand reçu' eq $Type + and 'EUR' eq $Devise + and 'Non compensé' eq $Etat + ) { + $invoice = next_invoice( ) ; + $nb_invoice++ ; + $print_details and print ( "[$invoice] [$Date] [$Heure] [$Fuseau_horaire] [$Nom] [$Type] [$Etat] [$Devise] [$Montant] [$Numero_davis_de_reception] [$Solde]\n" ) ; + } + + $action->{ 'invoice' } = $invoice ; + if ( $bnc ) { + my $FR_flag = '' ; + $FR_flag = ' FR' if $Pays eq 'France' ; + my $IND_flag = '' ; + $IND_flag = ' IND' if ('imapsync usage' eq $Nom_Option_1 and 'individual' eq $Valeur_Option_1 ) ; + my $SUPPORT_flag = '' ; + $SUPPORT_flag = ' support' if ( 'imapsync support' eq $Titre_de_l_objet ) ; + #print "FE $invoice$FR_flag$IND_flag\n" ; + #printf( "%.2f [EUR %.2f]\n", $Montant, $MontantEUR ) ; + print "FE $invoice$FR_flag$IND_flag imapsync$SUPPORT_flag $Nom\n" ; + print "[$Date]$FR_flag$IND_flag $MontantEUR $Devise \n" ; + } +} + +sub build_invoice { + my $invoice = shift ; + + return if ! $invoice ; + + my $action = $action_of_invoice{ $invoice } ; + my $refund = '' ; + $refund = 'REFUND ' if $invoice_refund{ $invoice } ; + my %action = %$action if $action ; + #print Data::Dumper->Dump( [$action] ) ; + + my( $Date, $Heure, $Nom, $Type, $Etat, $Devise, $Hors_taxe, $Commission, $Net, + $De_l_adresse_email, $A_l_adresse_email, $N_de_transaction, $Titre_de_l_objet, + $TVA, $Nom_Option_1, $Valeur_Option_1, $N_de_transaction_de_reference, + $Adresse_1, $Adresse_2_district_quartier, $Ville, + $Etat_Province, $Code_postal, $Pays, $line_number, $line_csv, $file_csv, + $Nom_Option_2, $Option_2_Valeur ) + = @action{ ( 'Date', 'Heure', 'Nom', 'Type', 'Etat', 'Devise', 'Hors taxe', 'Commission', 'Net', + "De l'adresse email", "A l'adresse email", 'N° de transaction', "Titre de l'objet", + 'TVA', 'Nom Option 1', 'Valeur Option 1', 'Nº de transaction de référence', + 'Adresse 1', 'Adresse 2/district/quartier', 'Ville', + 'Etat/Province/Région/Comté/Territoire/Préfecture/République', 'Code postal', 'Pays', 'line_number', 'line_csv', 'file_csv', + 'Nom Option 2', 'Option 2 Valeur' ) } ; + + my( $Etat_Province1 ) = @action{ ( 'Etat/Province/Région/Comté/Territoire/Préfecture/République' ) } ; + my( $Etat_Province2 ) = @action{ ( 'État/Province/Région/Comté/Territoire/Préfecture/République' ) } ; + + $Etat_Province = @action{ ( 'Etat/Province/Région/Comté/Territoire/Préfecture/République' ) } + || @action{ ( 'État/Province/Région/Comté/Territoire/Préfecture/République' ) } + || '' ; + + #print "$Hors_taxe $Devise\n" ; + my $Hors_taxe_num = $Hors_taxe ; + $Hors_taxe_num =~ s{,}{.} ; + if ($Hors_taxe_num > 100) { + print "invoice $invoice $Hors_taxe_num > 100\n" ; + #return() ; + } + + my ( $email_message_header, $email_message_body ) + = build_email_message( $Date, $Nom, $De_l_adresse_email, $invoice, $Titre_de_l_objet ) ; + if ( $write_invoices and ! invoice_sent( $dir_invoices, $invoice, $De_l_adresse_email ) ) { + write_email_message( $dir_invoices, $invoice, + $email_message_header, $email_message_body, + $De_l_adresse_email) ; + write_csv_info( $dir_invoices, $invoice, $file_csv, $line_number, $line_csv ) ; + } + + + + #print "==== $invoice $refund=================================================" ; + #print $email_message ; + + my( + $clientAdrA, + $clientAdrB, + $clientAdrC, + $clientAdrD, + $clientAdrE, + $clientAdrF, + ) + = build_address( + $Nom, + $Adresse_1, + $Adresse_2_district_quartier, + $Ville, + $Code_postal, + $Etat_Province, + $Pays, + ) ; + + foreach my $str ( + $De_l_adresse_email, + $Nom, + $clientAdrA, + $clientAdrB, + $clientAdrC, + $clientAdrD, + $clientAdrE, + $clientAdrF, + ) { + $str =~ s{#}{\\#}g ; + $str =~ s{_}{\\_}g ; + $str =~ s{&}{\\&}g ; + } + + my ( $clientTypeEN, $clientTypeFR ) = client_type( $Nom_Option_1, $Valeur_Option_1 ) ; + + my $quantity = '1' ; + + my ( + $descriptionFR, + $descriptionEN, + $usageFR, + $usageEN, + ) + = description_stuff( $Titre_de_l_objet, $clientTypeEN ) ; + + my ( + $priceHT, + $tvaFR, + $tvaEN, + $priceTVA, + $priceTTC, + $messageTVAFR, + $messageTVAEN, + $priceTTCusd, + $HTorTTC + ) + = tva_stuff( $clientTypeEN, $Pays, $Hors_taxe, $Devise, $Titre_de_l_objet ) ; + + my $object_type = object_type( $Titre_de_l_objet ) ; + + my ( $urlSrc, $urlExe ) = download_urls( $Date, $object_type ) ; + #print "ZZZ $object_type ( $urlSrc, $urlExe )\n" ; + + my ( $Nom1 ) = cut( $Nom, 42 ) ; + + my $clientVAT = '' ; + + if ( ( 'VAT if professional in Europe' eq $Nom_Option_2 ) and $Option_2_Valeur ) { + $clientVAT = $Option_2_Valeur ; + } + + my $tex_variables = qq{ +%% Begin input from paypal_bilan $VERSION +\\providecommand{\\invoiceNumber}{$invoice} +\\providecommand{\\clientName}{$Nom1} +\\providecommand{\\clientEmail}{$De_l_adresse_email} +\\providecommand{\\clientAdrA}{$clientAdrA} +\\providecommand{\\clientAdrB}{$clientAdrB} +\\providecommand{\\clientAdrC}{$clientAdrC} +\\providecommand{\\clientAdrD}{$clientAdrD} +\\providecommand{\\clientAdrE}{$clientAdrE} +\\providecommand{\\clientAdrF}{$clientAdrF} +\\providecommand{\\clientVAT}{$clientVAT} +\\providecommand{\\invoiceDate}{$Date} +\\providecommand{\\invoiceHour}{$Heure} + +\\providecommand{\\descriptionFR}{$descriptionFR} +\\providecommand{\\descriptionEN}{$descriptionEN} +\\providecommand{\\usageFR}{$usageFR} +\\providecommand{\\usageEN}{$usageEN} +\\providecommand{\\quantity}{$quantity} + +\\providecommand{\\priceHT}{$priceHT} +\\providecommand{\\tvaFR}{$tvaFR} +\\providecommand{\\tvaEN}{$tvaEN} +\\providecommand{\\priceTVA}{$priceTVA} +\\providecommand{\\HTorTTC}{$HTorTTC} +\\providecommand{\\priceTTC}{$priceTTC} +\\providecommand{\\priceTTCusd}{$priceTTCusd} +\\providecommand{\\messageTVAFR}{$messageTVAFR} +\\providecommand{\\messageTVAEN}{$messageTVAEN} +\\providecommand{\\urlSrc}{\\url{$urlSrc}} +\\providecommand{\\urlExe}{\\url{$urlExe}} +%% End input from paypal_bilan +} ; + + my $tex_variables_utf8 = to_utf8( { -string => $tex_variables, -charset => 'ISO-8859-1' } ) ; + + print $tex_variables_utf8 if $debug_invoice_utf8 ; + print $tex_variables if $debug_invoice ; + + #print "$invoice ", invoice_sent( $dir_invoices, $invoice, $De_l_adresse_email ), "\n" ; + if ( $write_invoices and ! invoice_sent( $dir_invoices, $invoice, $De_l_adresse_email ) ) { + write_tex_variables_file( $dir_invoices, $invoice, $Date, $tex_variables_utf8 ) ; + } + +} + +sub description_stuff { + my ( $object, $clientTypeEN ) = @_ ; + + my $object_type = object_type( $object ) ; + + my ( $descriptionFR, $descriptionEN ) ; + if ( 'software' eq $object_type ) { + $descriptionFR = 'Logiciel imapsync. Tous droits cédés.' ; + $descriptionEN = '(Imapsync software. All rights conceded.)' ; + } + + my ( $usageFR, $usageEN ) ; + if ( 'professional' eq $clientTypeEN + and 'software' eq $object_type ) { + $usageFR = 'Usage à titre professionnel.' ; + $usageEN = '(professional usage.)' ; + } + + if ( 'individual' eq $clientTypeEN + and 'software' eq $object_type ) { + $usageFR = 'Usage à titre individuel.' ; + $usageEN = '(individual usage.)' ; + } + + if ( 'support' eq $object_type ) { + $descriptionFR = 'Support sur le logiciel imapsync.' ; + $descriptionEN = '(Imapsync support.)' ; + $usageFR = '' ; + $usageEN = '' ; + } + return( $descriptionFR, $descriptionEN, $usageFR, $usageEN ) ; +} + + + +sub object_type { + my $object = shift ; + + if ( 'imapsync' eq $object + or 'imapsync.exe' eq $object + or 'imapsync source' eq $object + or 'imapsync source code' eq $object + ) { + return( 'software' ) ; + }elsif ( 'imapsync support' eq $object ) { + return( 'support' ) ; + } +} + +sub build_email_message { + + my ( $date, $name, $email, $invoice, $objet ) = @_ ; + + my $object_type = object_type( $objet ) ; + + my $message_header_software = qq{X-imapsync: invoice $invoice for imapsync software +From: Gilles LAMIRAL +Bcc: gilles\@lamiral.info +Subject: [imapsync invoice] $invoice ($date) for imapsync software +Disposition-Notification-To: Gilles LAMIRAL +} ; + + my $message_header_support = qq{X-imapsync: invoice $invoice for imapsync support +From: Gilles LAMIRAL +Bcc: gilles\@lamiral.info +Subject: [imapsync invoice] $invoice ($date) for imapsync support +Disposition-Notification-To: Gilles LAMIRAL +} ; + + my $message_body_software = qq{ +Hello $name, + +First of all, I'm sorry for the delay in getting back to you. + +You'll find in the attachment the invoice of imapsync +software you bought and paid (dd/mm/yyyy $date). +The invoice file is named facture_imapsync-${invoice}.pdf +This invoice is in PDF format, ready to be print. + +Should you need a hardcopy of this invoice, +I'll send it to you upon request by regular mail. + +As the law requires, this numeric invoice PDF file +is signed with my private gpg key. + +The resulting gpg signature is in the file named +facture_imapsync-${invoice}.pdf.asc +you will also find in the attachment. + +You can check I (Gilles LAMIRAL) really did generate +this invoice with the following command line: + + gpg --verify facture_imapsync-${invoice}.pdf.asc facture_imapsync-${invoice}.pdf + +or any other gpg graphical tool. + +Once more, thank you for buying and using imapsync. + +Any feedback is welcome. + + +-- +Best Regards, 09 51 84 42 42 +Gilles Lamiral. France, Baulon (35580) 06 20 79 76 06 +} ; + + + my $message_body_support = qq{ +Hello $name, + +First of all, I'm sorry for the delay in getting back to you. + +You'll find in the attachment the invoice of imapsync +support you bought and paid (dd/mm/yyyy $date). +The invoice file is named facture_imapsync-${invoice}.pdf +This invoice is in PDF format, ready to be print. + +Should you need a hardcopy of this invoice, +I'll send it to you upon request by regular mail. + +As the law requires, this numeric invoice PDF file +is signed with my private gpg key. + +The resulting gpg signature is in the file named +facture_imapsync-${invoice}.pdf.asc +you will also find in the attachment. + +You can check I (Gilles LAMIRAL) really did generate +this invoice with the following command line: + + gpg --verify facture_imapsync-${invoice}.pdf.asc facture_imapsync-${invoice}.pdf + +or any other gpg graphical tool. + +Once more, thank you for buying imapsync support. + +Any feedback is welcome. + +-- +Best Regards, 09 51 84 42 42 +Gilles Lamiral. France, Baulon (35580) 06 20 79 76 06 +} ; + + + + + my $message_body_blabla = qq{ +Here is the fingerprint of my public key +pub 1024D/FDA2B3DC 2002-05-08 + Key fingerprint = 7906 F53D 0D62 0C67 304A 4CF0 6928 869B FDA2 B3DC +uid Gilles LAMIRAL +sub 1024g/A2C4CB42 2002-05-08 + +Of course the verification doesn't prove anything until +all the following conditions are met: +- you met me, +- I agree that the fingerprint above is really mine +- I prove I'm Gilles LAMIRAL with an official paper. + +Normally we won't have to verify anything unless +I disagree with this invoice and the payment +you made for imapsync. +} ; + + my ( $message_header, $message_body ) ; + if ( 'support' eq $object_type ) { + $message_header = $message_header_support ; + $message_body = $message_body_support ; + }elsif ( 'software' eq $object_type ) { + $message_header = $message_header_software ; + $message_body = $message_body_software ; + } + return( $message_header, $message_body ) ; + +} + +sub write_csv_info { + + my( $dir_invoices, $invoice, $file_csv, $line_number, $line_csv ) = @_ ; + + open( CSVINFO, "> $dir_invoices/$invoice/csv_info.txt") or die ; + print CSVINFO join( "\n", $file_csv, $line_number, $line_csv ) ; + close( CSVINFO ) ; + +} + +sub invoice_sent { + + my ( $dir_invoices, $invoice, $email_address ) = @_ ; + + return( 1 ) if ( -f "$dir_invoices/$invoice/SENT_TO_$email_address" ) ; + return( 0 ) ; + +} + +sub write_email_message { + my ( $dir_invoices, $invoice, $message_header, $message_body, $email_address ) = @_ ; + + my $message_body_utf8 = to_utf8({ -string => $message_body, -charset => 'ISO-8859-1' }); + + mkdir( "$dir_invoices/$invoice" ) or die if ! -d "$dir_invoices/$invoice" ; + + open( HEADER, "> $dir_invoices/$invoice/facture_message_header.txt") or die ; + print HEADER $message_header ; + close( HEADER ) ; + + open( BODY, "> $dir_invoices/$invoice/facture_message_body.txt") or die ; + print BODY $message_body_utf8 ; + close( BODY ) ; + + open( ADDRESS, "> $dir_invoices/$invoice/email_address.txt") or die ; + print ADDRESS "$email_address\n" ; + close( ADDRESS ) ; +} + + +sub write_tex_variables_file { + my ( $dir_invoices, $invoice, $date_jjSmmSaaaa, $tex_variables_utf8 ) = @_ ; + + mkdir( "$dir_invoices/$invoice" ) or die if ! -d "$dir_invoices/$invoice" ; + open( FILE, "> $dir_invoices/$invoice/imapsync_var.tex") or die ; + print FILE $tex_variables_utf8 ; + close( FILE ) ; + + if ( ! -f "$dir_invoices/$invoice/imapsync_var_manual.tex" ) { + open( FILE, "> $dir_invoices/$invoice/imapsync_var_manual.tex") or die ; + print FILE "%% $0 created file +%% Can be used to override imapsync_var.tex definitions\n" ; + print FILE $tex_variables_utf8 ; + close( FILE ) ; + } + +} + +sub download_urls { + my $date_jjSmmSaaaa = shift ; + my $object_type = shift ; + + my $date_aaaa_mm_jj = date_aaaa_mm_jj( $date_jjSmmSaaaa ) ; + #print "$date_aaaa_mm_jj $date_jjSmmSaaaa $object_type\n" ; + my ( $urlSrc, $urlExe ) ; + + if ('2011_05_01' le $date_aaaa_mm_jj + and 'software' eq $object_type ) { + $urlSrc = 'http://ks.lamiral.info/imapsync/paypal_return.shtml' ; + $urlExe = '' ; + return( $urlSrc, $urlExe ) ; + } + + if ('2011_05_01' le $date_aaaa_mm_jj + and 'support' eq $object_type ) { + $urlSrc = 'http://ks.lamiral.info/imapsync/paypal_return_support.shtml' ; + $urlExe = '' ; + return( $urlSrc, $urlExe ) ; + } + + if ('2011_03_24' le $date_aaaa_mm_jj) { + $urlSrc = 'http://www.linux-france.org/prj/imapsync/paypal_return.shtml' ; + $urlExe = '' ; + return( $urlSrc, $urlExe ) ; + } + if ('2011_02_21' le $date_aaaa_mm_jj) { + $urlSrc = 'http://www.linux-france.org/depot/2011_02_21/OUMbo7/' ; + $urlExe = 'http://www.linux-france.org/depot/2011_02_21/rHSVNs/' ; + return( $urlSrc, $urlExe ) ; + } + if ('2011_01_18' le $date_aaaa_mm_jj) { + $urlSrc = 'http://www.linux-france.org/depot/2011_01_18/zPRRNt/' ; + $urlExe = 'http://www.linux-france.org/depot/2011_01_18/FO1QzG/' ; + return( $urlSrc, $urlExe ) ; + } + if ('2011_01_18' le $date_aaaa_mm_jj) { + $urlSrc = 'http://www.linux-france.org/depot/2010_11_28/SiNdlZ/' ; + $urlExe = 'http://www.linux-france.org/depot/2010_11_28/R3ZAyr/' ; + return( $urlSrc, $urlExe ) ; + } + $urlSrc = 'http://www.linux-france.org/depot/2010_11_08/X2PWMe/' ; + $urlExe = 'http://www.linux-france.org/depot/2010_11_08/ZZ7zSc/' ; + return( $urlSrc, $urlExe ) ; +} + +sub date_aaaa_mm_jj { + my $date_jjSmmSaaaa = shift ; + + if ( $date_jjSmmSaaaa =~ m{(\d\d)/(\d\d)/(\d\d\d\d)} ) { + my( $jj, $mm, $aaaa ) = ( $1, $2, $3 ) ; + return( join( '_', $aaaa, $mm, $jj ) ) ; + }else{ + return( '9999_12_31' ) ; + } +} + + +sub tva_line { + my( $Devise, $Montant2, $Pays, $Nom_Option_1, $Valeur_Option_1, $Titre_de_l_objet ) = @_ ; + my( $montant_HT_EUR_exo, $montant_HT_EUR_ass, $montant_TVA_EUR ) = ( 0, 0, 0 ) ; + + my( $montant_HT_EUR_sup, $montant_TVA_EUR_sup ) = ( 0, 0 ) ; + + $Montant2 = $Montant2/$usdeur if 'USD' eq $Devise ; + + if ( 'imapsync' eq $Titre_de_l_objet + or 'imapsync.exe' eq $Titre_de_l_objet + or 'imapsync source' eq $Titre_de_l_objet + or 'imapsync source code' eq $Titre_de_l_objet + + ) { + if ( + ( 'imapsync usage' eq $Nom_Option_1 and 'individual' eq $Valeur_Option_1 ) + or + ( 'France' eq $Pays ) + ) { + $montant_HT_EUR_ass = $Montant2 / 1.196 ; + $montant_TVA_EUR = $Montant2 / 1.196 * 0.196 ; + $debug_dev and print "$Montant2 $Pays $Valeur_Option_1\n" ; + }else{ + $montant_HT_EUR_exo = $Montant2 ; + } + } + + if ( 'imapsync support' eq $Titre_de_l_objet ) { + #print "ZZZZ $Titre_de_l_objet $Montant2\n" ; + $montant_HT_EUR_sup = $Montant2 / 1.196 ; + $montant_TVA_EUR_sup = $Montant2 / 1.196 * 0.196 ; + } + + + return( $montant_HT_EUR_exo, $montant_HT_EUR_ass, $montant_TVA_EUR, $montant_HT_EUR_sup, $montant_TVA_EUR_sup ) ; +} + + + +sub tva_stuff { + my( $clientTypeEN, $Pays, $Hors_taxe, $Devise, $Titre_de_l_objet ) = @_ ; + + my $priceTTCusd = '' ; + $Hors_taxe =~ s{,}{.} ; + + if ( $Devise eq 'USD' ) { + $priceTTCusd = "(usd $Hors_taxe)" ; + $Hors_taxe = ( $Hors_taxe/$usdeur ) ; + } + + my ( + $priceHT, + $tvaFR, + $tvaEN, + $priceTVA, + $priceTTC, + $messageTVAFR, + $messageTVAEN, + $HTorTTC + ) ; + + if ( ( 'individual' eq $clientTypeEN) + or + ( 'France' eq $Pays ) + or + ( 'imapsync support' eq $Titre_de_l_objet ) + ) { + $priceHT = sprintf('%2.2f', $Hors_taxe/1.196) ; + $tvaFR = '19,60\%'; + $tvaEN = ''; + $priceTVA = sprintf('%2.2f', $Hors_taxe/1.196*0.196) ; + $priceTTC = sprintf('%2.2f', $Hors_taxe) ; + $HTorTTC = 'TTC' ; + $messageTVAFR = '' ; + $messageTVAEN = '' ; + }else{ + $priceHT = sprintf('%2.2f', $Hors_taxe) ; + $tvaFR = '' ; + $tvaEN = '' ; + $priceTVA = 'néant (none)' ; + $priceTTC = $priceHT ; + $HTorTTC = 'HT' ; + $messageTVAFR = 'Exonération de TVA, articles 262 1-2 et ter du Code Général des Impôts'; + $messageTVAEN = '(VAT tax-exempt, articles 262 1-2 and ter of French General Tax Code)'; + } + foreach my $price ( $priceHT, $priceTVA, $priceTTC, $priceTTCusd ) { + #print "[$price]\n" ; + $price =~ s{\.}{, } ; + } + return( + $priceHT, + $tvaFR, + $tvaEN, + $priceTVA, + $priceTTC, + $messageTVAFR, + $messageTVAEN, + $priceTTCusd, + $HTorTTC + ) ; +} + +sub client_type { + my ( $Nom_Option_1, $Valeur_Option_1 ) = @_ ; + + my ( $clientTypeEN, $clientTypeFR ) = ( 'professional', 'professionnel' ) ; + + if ('imapsync usage' eq $Nom_Option_1 and 'individual' eq $Valeur_Option_1 ) { + $clientTypeEN = 'individual' ; + $clientTypeFR = 'individuel' ; + }elsif ('imapsync usage' eq $Nom_Option_1 and 'professional' eq $Valeur_Option_1 ) { + $clientTypeEN = 'professional' ; + $clientTypeFR = 'professionnel' ; + } + return( $clientTypeEN, $clientTypeFR ) ; +} + +sub build_address { + my( + $Nom, + $Adresse_1, + $Adresse_2_district_quartier, + $Ville, + $Code_postal, + $Etat_Province, + $Pays, + ) = @_ ; + + my $addr = " +=========================================================== +Nom $Nom +Adresse_1 $Adresse_1 +Adresse_2_district_quartier $Adresse_2_district_quartier +Ville Code_postal $Ville $Code_postal +Etat_Province $Etat_Province +Pays $Pays +" ; + #print $addr ; + + my @address ; + $Nom = '' if ( $Nom =~ m/^\s+$/ ) ; + my( $Nom1, $Nom2 ) = cut( $Nom, 42 ) ; + push( @address, $Nom1 ) if $Nom1 ; + #push( @address, $Nom2 ) if $Nom2 ; + push( @address, $Adresse_1 ) if $Adresse_1 ; + push( @address, $Adresse_2_district_quartier ) if $Adresse_2_district_quartier ; + push( @address, "$Ville $Code_postal" ) if ( $Ville or $Code_postal ); + push( @address, $Etat_Province ) if $Etat_Province ; + push( @address, $Pays, ) if $Pays ; + + + my $clientAdrA = shift( @address ) || '' ; + my $clientAdrB = shift( @address ) || '' ; + my $clientAdrC = shift( @address ) || '' ; + my $clientAdrD = shift( @address ) || '' ; + my $clientAdrE = shift( @address ) || '' ; + my $clientAdrF = shift( @address ) || '' ; + +$addr = " +[$clientAdrA] +[$clientAdrB] +[$clientAdrC] +[$clientAdrD] +[$clientAdrE] +[$clientAdrF] +"; + #print $addr ; + + return( + $clientAdrA, + $clientAdrB, + $clientAdrC, + $clientAdrD, + $clientAdrE, + $clientAdrF, + ) ; +} + + +sub half { + my $string = shift ; + my $half = int( lenght( $string ) / 2 ) ; + # TO BE DONE + +} + +sub tests_half { + my( $aa, $bb ) = half( 'aa bb' ) ; + ok( 'aa' eq $aa, 'half: aa' ) ; + ok( 'bb' eq $bb, 'half: bb' ) ; +} + +sub cut { + my $string = shift ; + my $offset = shift ; + + return( $string, '' ) if length( $string ) < $offset ; + my $first = substr( $string, 0, $offset ) ; + my $last = substr( $string, $offset ) ; + return( $first, $last ) ; +} + +sub tests_cut { + my( $aa, $bb ) = cut("123456789", 4 ) ; + ok( '1234' eq $aa, 'cut 123456789 4 => first 1234' ) ; + ok( '56789' eq $bb, 'cut 123456789 4 => last 56789' ) ; +} diff --git a/W/paypal_reply/paypal_build_invoices b/W/paypal_reply/paypal_build_invoices index 3c7ef7e..d6f7159 100755 --- a/W/paypal_reply/paypal_build_invoices +++ b/W/paypal_reply/paypal_build_invoices @@ -1,13 +1,12 @@ #!/bin/sh -# $Id: paypal_build_invoices,v 1.36 2012/08/10 22:37:18 gilles Exp gilles $ +# $Id: paypal_build_invoices,v 1.40 2012/10/03 14:29:07 gilles Exp gilles $ # usage: sh paypal_build_invoices /g/var/paypal_invoices/???? lyx -e latex /home/gilles/public_html/AGIL/factures/000/facture_imapsync-000.lyx cp /home/gilles/public_html/AGIL/factures/000/facture_imapsync-000.tex /g/var/paypal_invoices/ -set -x #/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 147 /g/paypal/paypal_2010_11_complet.csv #/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 214 /g/paypal/paypal_2010_12_complet.csv #/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 294 /g/paypal/paypal_2011_01_complet.csv @@ -29,7 +28,10 @@ set -x #/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 1654 /g/paypal/paypal_2012_05_complet.csv #/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 1743 /g/paypal/paypal_2012_06_complet.csv #/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 1824 /g/paypal/paypal_2012_07_complet.csv -/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 1891 /g/paypal/paypal_2012_08_complet.csv +#/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 1891 /g/paypal/paypal_2012_08_complet.csv +#/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 1963 /g/paypal/paypal_2012_09_complet.csv +/g/public_html/imapsync/W/paypal_reply/paypal_bilan --write_invoices --first_in 2047 /g/paypal/paypal_2012_10_complet.csv + : /g/public_html/imapsync/W/paypal_reply/paypal_bilan --first_in 147 /g/paypal/paypal_2010_11_complet.csv : /g/public_html/imapsync/W/paypal_reply/paypal_bilan --first_in 214 /g/paypal/paypal_2010_12_complet.csv @@ -53,6 +55,9 @@ set -x : /g/public_html/imapsync/W/paypal_reply/paypal_bilan --first_in 1743 /g/paypal/paypal_2012_06_complet.csv : /g/public_html/imapsync/W/paypal_reply/paypal_bilan --first_in 1824 /g/paypal/paypal_2012_07_complet.csv : /g/public_html/imapsync/W/paypal_reply/paypal_bilan --first_in 1891 /g/paypal/paypal_2012_08_complet.csv +: /g/public_html/imapsync/W/paypal_reply/paypal_bilan --first_in 1963 /g/paypal/paypal_2012_09_complet.csv +set -x +: /g/public_html/imapsync/W/paypal_reply/paypal_bilan --first_in 2047 /g/paypal/paypal_2012_10_complet.csv set +x diff --git a/W/test2.bat b/W/test2.bat index e9242f9..735b459 100644 --- a/W/test2.bat +++ b/W/test2.bat @@ -1,5 +1,5 @@ -REM $Id: test2.bat,v 1.10 2012/08/29 10:43:25 gilles Exp gilles $ +REM $Id: test2.bat,v 1.11 2012/10/05 09:15:55 gilles Exp gilles $ cd C:\msys\1.0\home\Admin\imapsync REM perl ./imapsync --host1 p --user1 tata --passfile1 secret.tata --host2 p --user2 titi --passfile2 secret.titi --delete2 --expunge2 --folder INBOX @@ -34,6 +34,10 @@ REM --host2 p --user2 titi --passfile2 secret.titi ^ REM --folder INBOX.useuid --useuid --debugcache --delete2 -perl ./imapsync --host1 p --user1 tata --passfile1 secret.tata ^ - --host2 imap.gmail.com --ssl2 --user2 gilles.lamiral@gmail.com --passfile2 secret.gilles_gmail ^ - --usecache --nofoldersizes --folder INBOX --regextrans2 "s(INBOX)([Gmail]/te*st)" +REM perl ./imapsync --host1 p --user1 tata --passfile1 secret.tata ^ +REM --host2 imap.gmail.com --ssl2 --user2 gilles.lamiral@gmail.com --passfile2 secret.gilles_gmail ^ +REM --usecache --nofoldersizes --folder INBOX --regextrans2 "s(INBOX)([Gmail]/te*st)" + +perl ./imapsync --host1 imap.gmail.com --port1 993 --ssl1 --host2 imap.bigs.dk --justconnect + +imapsync.exe --host1 imap.gmail.com --port1 993 --ssl1 --host2 imap.bigs.dk --justconnect diff --git a/imapsync b/imapsync index 72e3b23..d7110c2 100755 --- a/imapsync +++ b/imapsync @@ -20,7 +20,7 @@ Synchronise mailboxes between two imap servers. Good at IMAP migration. More than 44 different IMAP server softwares supported with success. -$Revision: 1.508 $ +$Revision: 1.516 $ =head1 SYNOPSIS @@ -106,7 +106,7 @@ The option list: [--expunge] [--expunge1] [--expunge2] [--uidexpunge2] [--delete2folders] [--delete2foldersonly] [--delete2foldersbutnot] [--subscribed] [--subscribe] [--subscribe_all] - [--nofoldersizes] + [--nofoldersizes] [--nofoldersizesatend] [--dry] [--debug] [--debugimap][--debugimap1][--debugimap2] [--timeout ] [--fast] @@ -403,7 +403,7 @@ Success stories reported with the following 44 imap servers - Qualcomm Worldmail (NT) - Rockliffe Mailsite 5.3.11, 4.5.6 - Samsung Contact IMAP server 8.5.0 - - Scalix v10.1, 10.0.1.3, 11.0.0.431 + - Scalix v10.1, 10.0.1.3, 11.0.0.431, 11.4.6 - SmarterMail, Smarter Mail 5.0 Enterprise, Smarter Mail 5.5 [host1]. - SunONE Messaging server 5.2, 6.0 (SUN JES - Java Enterprise System) - Sun Java(tm) System Messaging Server 6.2-2.05, 6.2-7.05, 6.3 @@ -515,7 +515,7 @@ Entries for imapsync: Feedback (good or bad) will often be welcome. -$Id: imapsync,v 1.508 2012/09/10 21:10:13 gilles Exp gilles $ +$Id: imapsync,v 1.516 2012/11/02 22:15:04 gilles Exp gilles $ =cut @@ -559,9 +559,9 @@ use constant { # global variables my( - $rcs, $pidfile, + $rcs, $pidfile, $pidfilelocking, $debug, $debugimap, $debugimap1, $debugimap2, $debugcontent, $debugflags, - $debugLIST, $debugsleep, + $debugLIST, $debugsleep, $debugdev, $nb_errors, $host1, $host2, $port1, $port2, $user1, $user2, $domain1, $domain2, @@ -580,8 +580,8 @@ my( $exitwhenover, $search, $skipheader, @useheader, - $skipsize, $allowsizemismatch, $foldersizes, $buffersize, - $delete, $delete2, + $skipsize, $allowsizemismatch, $foldersizes, $foldersizesatend, $buffersize, + $delete, $delete2, $delete2duplicates, $expunge, $expunge1, $expunge2, $uidexpunge2, $dry, $justfoldersizes, $authmd5, $authmd51, $authmd52, @@ -609,6 +609,8 @@ my( $h1_nb_msg_processed, $h1_nb_msg_start, $h1_bytes_start, $h2_nb_msg_start, $h2_bytes_start, + $h1_nb_msg_end, $h1_bytes_end, + $h2_nb_msg_end, $h2_bytes_end, $timeout, $timestart, $timestart_int, $timeend, $timediff, @@ -635,13 +637,16 @@ my( %h1, %h2, $checkselectable, $checkmessageexists, $expungeaftereach, + $abletosearch, + $showpasswords, + $fixslash2, ); # main program # global variables initialisation -$rcs = '$Id: imapsync,v 1.508 2012/09/10 21:10:13 gilles Exp gilles $ '; +$rcs = '$Id: imapsync,v 1.516 2012/11/02 22:15:04 gilles Exp gilles $ '; $total_bytes_transferred = 0; $total_bytes_skipped = 0; @@ -658,6 +663,9 @@ $h1_nb_msg_start = $h1_bytes_start = 0 ; $h2_nb_msg_start = $h2_bytes_start = 0 ; $h1_nb_msg_processed = $h1_bytes_processed = 0 ; +$h1_nb_msg_end = $h1_bytes_end = 0 ; +$h2_nb_msg_end = $h2_bytes_end = 0 ; + $nb_errors = 0; $max_msg_size_in_bytes = 0; @@ -682,26 +690,34 @@ unless(defined(&_SYSEXITS_H)) { } - # @ARGV will be eat by get_options() my @argv_copy = @ARGV; get_options(); +# $SIG{ INT } = \&catch_continue ; +$SIG{ INT } = \&catch_exit ; + +$timestart = time( ); +$timestart_int = int( $timestart ) ; +$timebefore = $timestart; + +my $timestart_str = localtime( $timestart ) ; +print "Transfer started at $timestart_str\n"; + $modules_version = defined($modules_version) ? $modules_version : 1; -# $SIG{ INT } = \&catch_continue ; $releasecheck = defined($releasecheck) ? $releasecheck : 1; my $warn_release = ($releasecheck) ? check_last_release() : ''; -$SIG{ INT } = \&catch_exit ; - # default values $tmpdir ||= File::Spec->tmpdir(); $pidfile ||= $tmpdir . '/imapsync.pid'; +$pidfilelocking = defined( $pidfilelocking ) ? $pidfilelocking : 0 ; + # allow Mail::IMAPClient 3.0.xx by default $allow3xx = defined($allow3xx) ? $allow3xx : 1; @@ -719,7 +735,8 @@ $relogin2 = defined( $relogin2 ) ? $relogin2 : 5 ; if ( $fast ) { # $useuid = 1 ; - $foldersizes = 0 ; + $foldersizes = 0 ; + $foldersizesatend = 0 ; } # Activate --usecache if --useuid is set and no --nousecache @@ -729,6 +746,11 @@ $cacheaftercopy = 1 if ( $usecache and ( ! defined( $cacheaftercopy ) ) ) ; $checkselectable = defined( $checkselectable ) ? $checkselectable : 1 ; $checkmessageexists = defined( $checkmessageexists ) ? $checkmessageexists : 1 ; $expungeaftereach = defined( $expungeaftereach ) ? $expungeaftereach : 1 ; +$abletosearch = defined( $abletosearch ) ? $abletosearch : 1 ; +$showpasswords = defined( $showpasswords ) ? $showpasswords : 0 ; +$fixslash2 = defined( $fixslash2 ) ? $fixslash2 : 1 ; + +$delete2duplicates = 1 if ( $delete2 and ( ! defined( $delete2duplicates ) ) ) ; print banner_imapsync(@argv_copy); @@ -808,7 +830,7 @@ if ( $delete2 and ! defined( $uidexpunge2 ) ) { } if ( $delete and $delete2 ) { - print "Warning: using --delete and --delete2 is almost always a bad idea, exiting imapsync\n" ; + print "Warning: using --delete and --delete2 together is almost always a bad idea, exiting imapsync\n" ; exit_clean( 4 ) ; } @@ -863,6 +885,7 @@ print "Info: will try to use $authmech2 authentication on host2\n"; $syncacls = (defined($syncacls)) ? $syncacls : 0; $foldersizes = (defined($foldersizes)) ? $foldersizes : 1; +$foldersizesatend = (defined($foldersizesatend)) ? $foldersizesatend : 1; $fastio1 = (defined($fastio1)) ? $fastio1 : 0; $fastio2 = (defined($fastio2)) ? $fastio2 : 0; @@ -904,9 +927,7 @@ $dry_message = "\t(not really since --dry mode)" if $dry ; my $imap1 = (); my $imap2 = (); -$timestart = time( ); -$timestart_int = int( $timestart ) ; -$timebefore = $timestart; + $debugimap1 and print "Host1 connection\n"; $imap1 = login_imap($host1, $port1, $user1, $domain1, $password1, @@ -1041,6 +1062,13 @@ $h2_prefix = get_prefix($imap2, $prefix2, "--prefix2"); print "Host1 separator and prefix: [$h1_sep][$h1_prefix]\n"; print "Host2 separator and prefix: [$h2_sep][$h2_prefix]\n"; +#my $h1_xlist_folders = $imap1->xlist_folders( ) ; +#my $h2_xlist_folders = $imap2->xlist_folders( ) ; +#require Data::Dumper ; +#print "Host1 xlist:\n", Data::Dumper->Dump([$h1_xlist_folders]) ; +#print "Host2 xlist:\n", Data::Dumper->Dump([$h2_xlist_folders]) ; + +#exit ; foreach my $h1_fold (@h1_folders_wanted) { my $h2_fold; @@ -1215,8 +1243,13 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { @$h1_fir_ref{@h1_msgs} = (undef); $debug and print "Host1 getting flags idate and sizes of folder [$h1_fold]\n" ; - $h1_fir_ref = $imap1->fetch_hash("FLAGS", "INTERNALDATE", "RFC822.SIZE", $h1_fir_ref) - if (@h1_msgs); + if ( $abletosearch ) { + $h1_fir_ref = $imap1->fetch_hash( "FLAGS", "INTERNALDATE", "RFC822.SIZE", $h1_fir_ref ) + if ( @h1_msgs ) ; + }else{ + $h1_fir_ref = $imap1->fetch_hash( '1:999999', "FLAGS", "INTERNALDATE", "RFC822.SIZE", $h1_fir_ref ) + if ( @h1_msgs ) ; + } $debug and print "Host1 getting flags idate and sizes of folder [$h1_fold] took ", timenext(), " s\n"; unless ($h1_fir_ref) { print @@ -1282,7 +1315,8 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { my $h2_msgs_duplicate_nb = scalar( @h2_msgs_duplicate ) ; $h2{ $h2_fold }{ 'duplicates_nb' } = $h2_msgs_duplicate_nb ; - $debug and print "Host2 selected: $h2_msgs_nb duplicates: $h2_msgs_duplicate_nb\n" ; + print "Host2 selected: $h2_msgs_nb, duplicates: $h2_msgs_duplicate_nb\n" + if ( $debug or $delete2duplicates or $h2_msgs_duplicate_nb ) ; $debug and print "Host2 whole time parsing headers took ", timenext(), " s\n"; $debug and print "++++ Verifying [$h1_fold] -> [$h2_fold]\n"; @@ -1310,6 +1344,29 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { } #next FOLDER ; + + if( $delete2duplicates ) { + my @h2_expunge ; + + foreach my $h2_msg ( @h2_msgs_duplicate ) { + print "msg $h2_fold/$h2_msg marked \\Deleted [duplicate] on host2 $dry_message\n" ; + push( @h2_expunge, $h2_msg ) if $uidexpunge2 ; + unless ( $dry ) { + $imap2->delete_message( $h2_msg ) ; + $h2_nb_msg_deleted += 1 ; + } + } + my $cnt = scalar @h2_expunge ; + if( @h2_expunge ) { + print "uidexpunge $cnt message(s) $dry_message\n" ; + $imap2->uidexpunge( \@h2_expunge ) if ! $dry ; + } + if ( $expunge2 ){ + print "Expunging host2 folder $h2_fold $dry_message\n" ; + $imap2->expunge( ) if ! $dry ; + } + } + if($delete2) { my @h2_expunge; foreach my $m_id (@h2_hash_keys_sorted_by_uid) { @@ -1318,7 +1375,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { my $h2_msg = $h2_hash{$m_id}{'m'}; my $h2_flags = $h2_hash{$m_id}{'F'} || ""; my $isdel = $h2_flags =~ /\B\\Deleted\b/ ? 1 : 0; - print "msg $h2_fold/$h2_msg marked \\Deleted on host2 [$m_id]\n" + print "msg $h2_fold/$h2_msg marked \\Deleted on host2 [$m_id] $dry_message\n" if ! $isdel; push(@h2_expunge, $h2_msg) if $uidexpunge2; unless ($dry or $isdel) { @@ -1327,16 +1384,8 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { } } } - foreach my $h2_msg (@h2_msgs_duplicate) { - print "msg $h2_fold/$h2_msg marked \\Deleted [duplicate] on host2\n"; - push(@h2_expunge, $h2_msg) if $uidexpunge2; - unless ($dry) { - $imap2->delete_message($h2_msg); - $h2_nb_msg_deleted += 1; - } - } foreach my $h2_msg ( @h2_msgs_delete2_not_in_cache ) { - print "msg $h2_fold/$h2_msg marked \\Deleted [not in cache] on host2\n"; + print "msg $h2_fold/$h2_msg marked \\Deleted [not in cache] on host2 $dry_message\n"; push(@h2_expunge, $h2_msg) if $uidexpunge2; unless ($dry) { $imap2->delete_message($h2_msg); @@ -1345,7 +1394,7 @@ FOLDER: foreach my $h1_fold (@h1_folders_wanted) { } my $cnt = scalar @h2_expunge ; if( @h2_expunge ) { - print "uidexpunge $cnt message(s)\n" ; + print "uidexpunge $cnt message(s) $dry_message\n" ; $imap2->uidexpunge( \@h2_expunge ) if ! $dry ; } if ($expunge2){ @@ -1508,14 +1557,17 @@ sub sync_flags { print "++++ End looping on each folder\n"; #print memory_consumption(); +if ( $foldersizesatend ) { + ( $h1_nb_msg_end, $h1_bytes_end ) = foldersizes( "Host1", $imap1, @h1_folders_wanted ) ; + ( $h2_nb_msg_end, $h2_bytes_end ) = foldersizes( "Host2", $imap2, @h2_folders_from_1_wanted ) ; +} -$imap1->logout(); -$imap2->logout(); +$imap1->logout( ) ; +$imap2->logout( ) ; - -stats(); -exit_clean(1) if( $nb_errors ); -exit_clean(0); +stats( ) ; +exit_clean( 1 ) if ( $nb_errors ) ; +exit_clean( 0 ) ; # END of main program @@ -1596,6 +1648,8 @@ Authen::NTLM)) sub command_line_nopassword { my @argv_copy = @_; my @argv_nopassword; + + return("@argv_copy") if $showpasswords ; while (@argv_copy) { my $arg = shift(@argv_copy); # option name or value if ($arg =~ m/-password[12]/) { @@ -1616,7 +1670,13 @@ sub tests_command_line_nopassword { ok('--password1 MASKED' eq command_line_nopassword(qw{ --password1 secret1}), 'command_line_nopassword --password1'); ok('--blabla --password1 MASKED --blibli' eq command_line_nopassword(qw{ --blabla --password1 secret1 --blibli }), 'command_line_nopassword --password1 --blibli'); - + $showpasswords = 1 ; + ok('' eq command_line_nopassword(), 'command_line_nopassword void'); + ok('--blabla' eq command_line_nopassword('--blabla'), 'command_line_nopassword --blabla'); + #print command_line_nopassword((qw{ --password1 secret1 })), "\n"; + ok('--password1 secret1' eq command_line_nopassword(qw{ --password1 secret1}), 'command_line_nopassword --password1'); + ok('--blabla --password1 secret1 --blibli' + eq command_line_nopassword(qw{ --blabla --password1 secret1 --blibli }), 'command_line_nopassword --password1 --blibli'); } @@ -1635,7 +1695,7 @@ sub catch_exit { my $signame = shift ; print "\nGot a SIG$signame!\n" ; stats( ) ; - exit_clean( ) ; + exit_clean( 6 ) ; } sub catch_continue { @@ -1877,7 +1937,7 @@ sub login_imap { $imap->Authuser($authuser); $imap->Password($password); } - + unless ( $authmech eq 'PREAUTH' or $imap->login( ) ) { my $info = "Failure: error login on [$host] with user [$user] auth" ; my $einfo = $imap->LastError || @{$imap->History}[-1] ; @@ -1923,8 +1983,8 @@ sub banner_imapsync { my @argv_copy = @_; my $banner_imapsync = join("", '$RCSfile: imapsync,v $ ', - '$Revision: 1.508 $ ', - '$Date: 2012/09/10 21:10:13 $ ', + '$Revision: 1.516 $ ', + '$Date: 2012/11/02 22:15:04 $ ', "\n",localhost_info(), "\n", "Command line used:\n", "$0 ", command_line_nopassword(@argv_copy), "\n", @@ -1941,35 +2001,38 @@ sub is_valid_directory { } -sub write_pidfile { - my $pidfile = shift; +sub write_pidfile { + my $pidfile = shift ; - print "PID file is $pidfile\n"; - if (-e $pidfile) { - print "$pidfile already exists, overwriting it\n"; - } - open(PIDFILE, ">$pidfile") or do { - print "Could not open $pidfile for writing"; - return undef; - }; + print "PID file is $pidfile\n" ; + if ( -e $pidfile and $pidfilelocking ) { + print "$pidfile already exists, another imapsync may be curently running. Aborting imapsync.\n" ; + exit( 8 ) ; + } + if ( -e $pidfile ) { + print "$pidfile already exists, overwriting it\n" ; + } + open( PIDFILE, ">$pidfile" ) or do { + print "Could not open $pidfile for writing" ; + return undef ; + } ; - print PIDFILE $PROCESS_ID; - close PIDFILE; - return($PROCESS_ID); -} + print PIDFILE $PROCESS_ID ; + close PIDFILE ; + return( $PROCESS_ID ) ; +} sub exit_clean { - my $status = shift; + my $status = shift ; $status = defined( $status ) ? $status : 1 ; - unlink($pidfile); - exit($status); + unlink( $pidfile ) ; + exit( $status ) ; } -sub die_clean { - - unlink($pidfile); - die @_; -} +sub die_clean { + unlink( $pidfile ) ; + die @_ ; +} sub missing_option { my ($option) = @_; @@ -2251,18 +2314,19 @@ sub folders_list_to_help { } -sub separator_invert { - # The separator we hope we'll never encounter: 00000000 - my $o_sep="\000"; +sub separator_invert { + # The separator we hope we'll never encounter: 00000000 + my $o_sep="\000" ; - my($h1_fold, $h1_sep, $h2_sep) = @_; + my($h1_fold, $h1_sep, $h2_sep) = @_ ; - my $h2_fold = $h1_fold; - $h2_fold =~ s@\Q$h2_sep@$o_sep@g; - $h2_fold =~ s@\Q$h1_sep@$h2_sep@g; - $h2_fold =~ s@\Q$o_sep@$h1_sep@g; - return($h2_fold); -} + my $h2_fold = $h1_fold ; + $h2_fold =~ s@\Q$h2_sep@$o_sep@g ; + $h2_fold =~ s@\Q$h1_sep@$h2_sep@g ; + $h2_fold =~ s@\Q$o_sep@$h1_sep@g ; + $h2_fold =~ s,/,_,g if( $fixslash2 and '/' ne $h2_sep and '/' eq $h1_sep ) ; + return( $h2_fold ) ; +} sub tests_imap2_folder_name { @@ -2278,6 +2342,7 @@ sep1:[$h1_sep] sep2:[$h2_sep] "; +$fixslash2 = 0 ; ok('' eq imap2_folder_name(''), 'imap2_folder_name: empty string'); ok('blabla' eq imap2_folder_name('blabla'), 'imap2_folder_name: blabla'); ok('spam.spam' eq imap2_folder_name('spam/spam'), 'imap2_folder_name: spam/spam'); @@ -2297,6 +2362,22 @@ ok('bla_bla' eq imap2_folder_name('bla bla'), 'imap2_folder_name: blabla [s, ,_, @regextrans2 = ('s,(.*),\U$1,'); ok('BLABLA' eq imap2_folder_name('blabla'), 'imap2_folder_name: blabla [s,\U(.*)\E,$1,]'); +$fixslash2 = 1 ; +@regextrans2 = ( ) ; +ok('' eq imap2_folder_name(''), 'imap2_folder_name: empty string'); +ok('blabla' eq imap2_folder_name('blabla'), 'imap2_folder_name: blabla'); +ok('spam.spam' eq imap2_folder_name('spam/spam'), 'imap2_folder_name: spam/spam -> spam.spam'); +ok('spam_spam' eq imap2_folder_name('spam.spam'), 'imap2_folder_name: spam.spam -> spam_spam'); +ok('spam.spam_spam' eq imap2_folder_name('spam/spam.spam'), 'imap2_folder_name: spam/spam.spam -> spam.spam_spam'); + +$h1_sep = '.'; +$h2_sep = '/'; +ok('' eq imap2_folder_name(''), 'imap2_folder_name: empty string'); +ok('blabla' eq imap2_folder_name('blabla'), 'imap2_folder_name: blabla'); +ok('spam.spam' eq imap2_folder_name('spam/spam'), 'imap2_folder_name: spam/spam -> spam.spam'); +ok('spam/spam' eq imap2_folder_name('spam.spam'), 'imap2_folder_name: spam.spam -> spam/spam'); +ok('spam.spam/spam' eq imap2_folder_name('spam/spam.spam'), 'imap2_folder_name: spam/spam.spam -> spam.spam/spam'); + } @@ -2359,7 +2440,11 @@ sub foldersizes { my $smax = 0 ; @$hash_ref{ @msgs } = ( undef ) if @msgs ; if ( $nb_msgs > 0 and @msgs ) { - $imap->fetch_hash("RFC822.SIZE",$hash_ref) or die_clean("$@") ; + if ( $abletosearch ) { + $imap->fetch_hash( "RFC822.SIZE", $hash_ref) or die_clean("$@" ) ; + }else{ + $imap->fetch_hash( '1:999999', "RFC822.SIZE", $hash_ref ) or die_clean( "$@" ) ; + } #print map {$hash_ref->{$_}->{"RFC822.SIZE"}, " "} keys %$hash_ref ; map { $stot += $hash_ref->{ $_ }->{ "RFC822.SIZE" } } keys %$hash_ref ; $smax = max( map { $hash_ref->{ $_ }->{ "RFC822.SIZE" } } keys %$hash_ref ) ; @@ -2647,7 +2732,15 @@ sub select_msgs { if ( defined( $msgs_all_hash_ref ) or ( ! defined( $maxage ) and ! defined( $minage ) and ! defined( $search ) ) ) { - @msgs = $imap->messages() ; + if ( $abletosearch ) { + $debugdev and print "Calling messages()\n" ; + @msgs = $imap->messages() ; + }else{ + $debugdev and print "Calling fetch_hash()\n" ; + @msgs = sort { $a <=> $b } keys( %{$imap->fetch_hash( '1:999999', "RFC822.SIZE")} ) ; + } + $debugdev and print "Done fetch_hash()\n" ; + if ( defined( $msgs_all_hash_ref ) ) { @msgs_all = @msgs ; @{ $msgs_all_hash_ref }{ @msgs_all } = () ; @@ -3474,38 +3567,37 @@ sub regexmess { return($string); } -sub bytes_display_string { - my ($bytes) = @_; +sub bytes_display_string { + my ( $bytes ) = @_ ; - if ($bytes < (1024 * 1024)) { - return sprintf("%.2f KiB", $bytes / 1024); - } elsif ($bytes < (1024 * 1024 * 1024)) { - return sprintf("%.2f MiB", $bytes / (1024 * 1024)); - } elsif ($bytes < (1024 * 1024 * 1024 * 1024)) { - return sprintf("%.2f GiB", $bytes / (1024 * 1024 * 1024)); - } elsif ($bytes < (1024 * 1024 * 1024 * 1024 * 1024)) { - return sprintf("%.2f TiB", $bytes / (1024 * 1024 * 1024 * 1024)); - } else { - return sprintf("%.2f PiB", $bytes / (1024 * 1024 * 1024 * 1024 * 1024)); - } + if ( abs( $bytes ) < ( 1024 * 1024 ) ) { + return sprintf( "%.2f KiB", $bytes / 1024) ; + } elsif ( abs( $bytes ) < ( 1024 * 1024 * 1024 ) ) { + return sprintf( "%.2f MiB", $bytes / (1024 * 1024) ) ; + } elsif ( abs( $bytes ) < ( 1024 * 1024 * 1024 * 1024) ) { + return sprintf("%.2f GiB", $bytes / (1024 * 1024 * 1024) ) ; + } elsif ( abs( $bytes ) < ( 1024 * 1024 * 1024 * 1024 * 1024) ) { + return sprintf( "%.2f TiB", $bytes / (1024 * 1024 * 1024 * 1024) ) ; + } else { + return sprintf( "%.2f PiB", $bytes / (1024 * 1024 * 1024 * 1024 * 1024) ) ; + } # if you have exabytes (EiB) of email to transfer, you have too much email } sub stats { - $timeend = time(); - $timediff = $timeend - $timestart; + $timeend = time( ); + $timediff = $timeend - $timestart ; - my $timestart_str = localtime( $timestart ) ; my $timeend_str = localtime( $timeend ) ; my $memory_consumption = memory_consumption( ) || 0 ; my $memory_ratio = ($max_msg_size_in_bytes) ? - sprintf('%.1f', $memory_consumption / $max_msg_size_in_bytes) : "NA"; + sprintf('%.1f', $memory_consumption / $max_msg_size_in_bytes) : "NA" ; - my $host1_reconnect_count = $imap1->Reconnect_counter() || 0; - my $host2_reconnect_count = $imap2->Reconnect_counter() || 0; + my $host1_reconnect_count = $imap1->Reconnect_counter() || 0 ; + my $host2_reconnect_count = $imap2->Reconnect_counter() || 0 ; - print "++++ Statistics\n"; + print "++++ Statistics\n" ; print "Transfer started on : $timestart_str\n"; print "Transfer ended on : $timeend_str\n"; printf( "Transfer time : %.1f sec\n", $timediff ) ; @@ -3542,9 +3634,17 @@ sub stats { printf ("Memory consumption : %.1f MiB\n", $memory_consumption / 1024 / 1024); print "Biggest message : $max_msg_size_in_bytes bytes\n"; # print "Memory/biggest message ratio : $memory_ratio\n"; - print "Detected $nb_errors errors\n\n"; + if ( $foldersizesatend and $foldersizes ) { + printf("Initial difference host2 - host1 : %s messages, %s bytes (%s)\n", $h2_nb_msg_start - $h1_nb_msg_start, + $h2_bytes_start - $h1_bytes_start, + bytes_display_string( $h2_bytes_start - $h1_bytes_start ) ) ; + printf("Final difference host2 - host1 : %s messages, %s bytes (%s)\n", $h2_nb_msg_end - $h1_nb_msg_end, + $h2_bytes_end - $h1_bytes_end, + bytes_display_string( $h2_bytes_end - $h1_bytes_end ) ) ; + } + print "Detected $nb_errors errors\n\n" ; - print $warn_release, "\n"; + print $warn_release, "\n" ; print thank_author(); } @@ -3579,6 +3679,7 @@ sub get_options { "debugimap!" => \$debugimap, "debugimap1!" => \$debugimap1, "debugimap2!" => \$debugimap2, + "debugdev!" => \$debugdev, "host1=s" => \$host1, "host2=s" => \$host2, "port1=i" => \$port1, @@ -3602,6 +3703,7 @@ sub get_options { "exclude=s" => \@exclude, "prefix1=s" => \$prefix1, "prefix2=s" => \$prefix2, + "fixslash2!" => \$fixslash2, "regextrans2=s" => \@regextrans2, "regexmess=s" => \@regexmess, "regexflag=s" => \@regexflag, @@ -3609,6 +3711,7 @@ sub get_options { "flagsCase!" => \$flagsCase, "delete!" => \$delete, "delete2!" => \$delete2, + "delete2duplicates!" => \$delete2duplicates, "delete2folders!" => \$delete2folders, "delete2foldersonly=s" => \$delete2foldersonly, "delete2foldersbutnot=s" => \$delete2foldersbutnot, @@ -3621,6 +3724,7 @@ sub get_options { "minage=i" => \$minage, "search=s" => \$search, "foldersizes!" => \$foldersizes, + "foldersizesatend!" => \$foldersizesatend, "dry!" => \$dry, "expunge!" => \$expunge, "expunge1!" => \$expunge1, @@ -3669,6 +3773,7 @@ sub get_options { "justlogin!" => \$justlogin, "tmpdir=s" => \$tmpdir, "pidfile=s" => \$pidfile, + "pidfilelocking!" => \$pidfilelocking, "releasecheck!" => \$releasecheck, "modules_version!" => \$modules_version, "usecache!" => \$usecache, @@ -3680,6 +3785,8 @@ sub get_options { "checkselectable!" => \$checkselectable, "checkmessageexists!" => \$checkmessageexists, "expungeaftereach!" => \$expungeaftereach, + "abletosearch!" => \$abletosearch, + "showpasswords!" => \$showpasswords, ); $debug and print "get options: [$opt_ret]\n"; @@ -3869,7 +3976,7 @@ sub check_last_release { } sub imapsync_version { - my $rcs = '$Id: imapsync,v 1.508 2012/09/10 21:10:13 gilles Exp gilles $ '; + my $rcs = '$Id: imapsync,v 1.516 2012/11/02 22:15:04 gilles Exp gilles $ '; $rcs =~ m/,v (\d+\.\d+)/; my $VERSION = ($1) ? $1: "UNKNOWN"; return($VERSION); @@ -3993,6 +4100,8 @@ Several options are mandatory. Required by Sun/iPlanet/Netscape IMAP servers to be able to use an administrative user --password1 : password for the user1. Dangerous, use --passfile1 +--showpasswords : shows passwords on output instead of "MASKED". + Useful to restart a complete run by just reading a log. --passfile1 : password file for the user1. Contains the password. --host2 : "destination" imap server. Mandatory. --port2 : port to connect on host2. Default is 143. @@ -4029,6 +4138,8 @@ Several options are mandatory. Will be created if it doesn't exist. Default is system specific and should be ok. --pidfile : the file where imapsync pid is written. +--pidfilelocking : Abort if pidfile already exists. Usefull to avoid + concurrent transfers on the same mailbox. --prefix1 : remove prefix to all destination folders (usually INBOX. for cyrus imap servers) you can use --prefix1 if your source imap server @@ -4058,6 +4169,9 @@ Several options are mandatory. they are really deleted unless --noexpunge is used. --delete2 : delete messages in host2 that are not in host1 server. Useful for backup or pre-sync. +--delete2duplicates : delete messages in host2 that are duplicates. + Works only without --useuid since duplicates are + detected with header part of each message. --delete2folders : Delete folders in host2 that are not in host1 server. For safety, first try it like this (it is safe): --delete2folders --dry --justfolders --nofoldersizes @@ -4105,6 +4219,8 @@ Several options are mandatory. host2 even if they are not subscribed on host1. --nofoldersizes : Do not calculate the size of each folder in bytes and message counts. Default is to calculate them. +--nofoldersizesatend : Do not calculate the size of each folder in bytes + and message counts at the end. Default is on. --justfoldersizes : exit after printed the folder sizes. --syncacls : synchronises acls (Access Control Lists). --nosyncacls : does not synchronise acls. This is the default. @@ -4668,7 +4784,7 @@ sub tests_debug { SKIP: { skip "No test in normal run" if ( not $tests_debug ); - tests_time_remaining( ) ; + tests_imap2_folder_name( ) ; } } @@ -4708,6 +4824,7 @@ sub tests { tests_cache_dir_fix( ) ; tests_filter_forbidden_characters( ) ; tests_cache_folder( ) ; + tests_time_remaining( ) ; } } diff --git a/index.shtml b/index.shtml index 0d88fd0..97c23ff 100644 --- a/index.shtml +++ b/index.shtml @@ -5,7 +5,7 @@ Imapsync: an IMAP migration tool ( release <!--#exec cmd="cat ./VERSION"--> ) - + @@ -46,7 +46,9 @@

imapsync software is a command line tool allowing incremental and recursive IMAP transfers from one mailbox to another, both anywhere on the internet or in your local network. "Incremental" means you can stop the transfer at any time -and restart it later efficiently. "Recursive" means all folders hierarchy can be copied. +and restart it later efficiently. "Recursive" means all folders hierarchy can be copied. +"Command line" means it's not a graphical tool, imapsync on Windows has to be run in a DOS box +(cmd.exe) or from a batch file.

imapsync is useful for imap account migration or imap account backup. @@ -95,7 +97,7 @@ total is 93 millions for 2011

New features or bugfixes since previous releases:

+
    +
  • 1.516
    • + +
  • Usability: Added host2 minus host1 statistic: number of messages and bytes. +One difference at the start and one at the end. +Useful to detect quickly a difference between host2 and host1. +Need both --foldersizes and --foldersizesatend options (which are on by default).
    • + +
  • Usability: Added --foldersizesatend. It gets the folders sizes at the end of the whole transfer. +Turned on by default (use --nofoldersizesatend to turn it off).
    • + +
  • Enhancement: Added option --delete2duplicates; it deletes messages in host2 that are duplicates. +--delete2duplicates is on when --delete2 is set unless --nodelete2duplicates is set too. +
    • + +
  • Usability: Added current date at the beginning of the run, +useful when imapsync doesn't end properly or hasn't finished yet.
    • + +
  • Enhancement: Added option --pidfilelocking; it aborts imapsync, when just launched, +in case another imapsync may be running. +Turned off by default
    • + +
  • Usability: Added option --showpasswords. it shows passwords on output instead of "MASKED". +Turned off by default. +Useful to restart a run by reading the log or to see what happen with special characters. +
    • + +
  • Bug fix: Added option --fixslash2 to avoid the 'Invalid mailbox name' error. +It acts when --sep2 is not character / and --sep1 is character / and host1 folder name contains --sep2 characters. +Turned on by default (--nofixslash2 to unable it).
    • + +
  • Enhancement: Added option --noabletosearch to allow the listing of messages without the imap "SEARCH ALL" command. +It's useful for playing with poor imap servers like Softalk 7.6.4 (8.6 is fine with SEARCH ALL).
    • + + +
+ +
  • 1.508
  • Usability: imapsync guesses and prints when it'll finish the transfer; added ETA after each copy (Estimated Time of Arrival)
  • Enhancement: Added --noexpungeaftereach to speedup --delete --expunge from Gmail.
  • Usability: Added Host1 or Host2 before "Nb messages" "Total size" with --foldersiszes (to facilitate parsing)
    • -
  • Bug fix: Previous fix about characters *|?:"<> in cache path was not complete.
    • +
  • Bug fix: Previous fix about characters *|?:"<> to _ in cache path was not complete.
@@ -213,7 +253,7 @@ any Unix, Linux, Windows, or Mac OS operating system.

-+ One year of imapsync updates without extra payment.
++ Two years of imapsync updates without extra payment.
+ 30 days money-back guarantee! (any reason qualifies)

@@ -234,7 +274,7 @@ Price 42 EUR is equal to around 50 USD, no problem to pay in USD (
-Enter your VAT if you're a European professional (for accounting). It's 2 letters followed by 11 digits, for example mine is FR74429303332. +Enter your VAT if you're a European professional (for accounting). It's 2 letters followed by 11 digits, for example mine is FR74429303332.
standalone imapsync.exe for 42 EUR

-+ One year of imapsync updates without extra payment.
++ Two years of imapsync updates without extra payment.
+ 30 days money-back guarantee! (any reason qualifies)

@@ -279,7 +319,7 @@ Price 42 EUR is equal to around 50 USD, no problem to pay in USD (
-Enter your VAT if you're a European professional (for accounting). It's 2 letters followed by 11 digits, for example mine is FR74429303332. +Enter your VAT if you're a European professional (for accounting). It's 2 letters followed by 11 digits, for example mine is FR74429303332.

-Enter your VAT if you're a European professional (for accounting). It's 2 letters followed by 11 digits, for example mine is FR74429303332. +Enter your VAT if you're a European professional (for accounting). It's 2 letters followed by 11 digits, for example mine is FR74429303332.
delivery postal address where the invoice will be needed (y since reedit is not easy (and forbidden by law).

-

Documentation

+

Documentation

Read the INSTALL file to know how to install imapsync on your system.

@@ -497,7 +537,7 @@ Don't hesitate to have a try, I will help you and make efforts to switch them to
  • Qualcomm Worldmail (NT) (http://www.eudora.com/worldmail/)
  • Rockliffe Mailsite 5.3.11, 4.5.6 (http://www.mailsite.com/)
  • Samsung Contact IMAP server 8.5.0
    • -
  • Scalix v10.1, 10.0.1.3, 11.0.0.431 (http://www.scalix.com/)
    • +
  • Scalix v10.1, 10.0.1.3, 11.0.0.431, 11.4.6 (http://www.scalix.com/)
  • SmarterMail, Smarter Mail 5.0 Enterprise, Smarter Mail 5.5 [host1]. (http://www.smartertools.com/)
  • SunONE Messaging server 5.2, 6.0 (SUN JES - Java Enterprise System) (http://www.oracle.com/)
  • Sun Java(tm) System Messaging Server 6.2-2.05, 6.2-7.05, 6.3 (http://www.oracle.com/)
    • @@ -576,7 +616,7 @@ alt="Viewable With Any Browser" /> This document last modified on -($Id: index.shtml,v 1.131 2012/09/11 22:07:18 gilles Exp gilles $) +($Id: index.shtml,v 1.136 2012/11/03 00:46:00 gilles Exp gilles $)

    diff --git a/tests.sh b/tests.sh index 12b1f59..16eceb7 100644 --- a/tests.sh +++ b/tests.sh @@ -1,6 +1,6 @@ #!/bin/sh -# $Id: tests.sh,v 1.201 2012/09/10 21:12:35 gilles Exp gilles $ +# $Id: tests.sh,v 1.205 2012/11/03 00:38:15 gilles Exp gilles $ # Example 1: # CMD_PERL='perl -I./Mail-IMAPClient-3.32/lib' sh -x tests.sh @@ -49,7 +49,7 @@ run_test() { if test x"$?" = x"0"; then echo "$1 passed" else - echo "$1 failed" >&2 + echo "$1 failed" fi } @@ -145,6 +145,10 @@ passwords_masked() { $CMD_PERL ./imapsync --host1 boumboum --password1 secret --justbanner | grep MASKED } +passwords_not_masked() { + $CMD_PERL ./imapsync --host1 boumboum --password1 secret --justbanner --showpasswords| grep secret +} + first_sync_dry() { $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 toto \ @@ -165,7 +169,7 @@ first_sync() { locallocal() { - can_send && sendtestmessage + #can_send && sendtestmessage $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 tata \ --passfile1 ../../var/pass/secret.tata \ @@ -194,6 +198,34 @@ pidfile() { ! test -f /var/tmp/imapsync.pid } +ll_pidfilelocking() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX --pidfile /var/tmp/imapsync_test_pidfilelocking.pid \ + --pidfilelocking --justconnect + echo "Exited $?" + ! test -f /var/tmp/imapsync_test_pidfilelocking.pid + touch /var/tmp/imapsync_test_pidfilelocking.pid + if $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX --pidfile /var/tmp/imapsync_test_pidfilelocking.pid \ + --pidfilelocking --justconnect; then + echo "Exited $?"; ! : + else + echo "Exited $?" + fi + + test -f /var/tmp/imapsync_test_pidfilelocking.pid + rm /var/tmp/imapsync_test_pidfilelocking.pid +} + + justbanner() { $CMD_PERL ./imapsync \ --justbanner @@ -887,6 +919,17 @@ ll_maxage_nonew() } +ll_nosearch_hack() +{ + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.few_emails --debugdev --debugimap1 --noabletosearch +} + + ll_newmessage() { can_send && sendtestmessage @@ -1070,7 +1113,7 @@ ll_regextrans2() --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ --justfolders \ - --nofoldersize \ + --nofoldersizes \ --regextrans2 's/yop/yoX/' \ --folder 'INBOX.yop.yap' } @@ -1087,7 +1130,7 @@ ll_regextrans2_downcase() --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ --justfolders \ - --nofoldersize \ + --nofoldersizes \ --regextrans2 's/(.*)\Q${h1_sep}\E(.+)$/$1${h2_sep}\L$2\E/' \ --folder 'INBOX.yop.YAP' --justfolders --debug --dry } @@ -1104,7 +1147,7 @@ ll_regextrans2_ucfirst() --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ --justfolders \ - --nofoldersize \ + --nofoldersizes \ --regextrans2 's/(.*)\Q${h1_sep}\E(.)(.+)$/$1${h2_sep}\u$2\L$3\E/' \ --folder 'INBOX.yop.YAP' --justfolders --debug --dry } @@ -1118,7 +1161,7 @@ ll_regextrans2_slash() --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ --justfolders \ - --nofoldersize \ + --nofoldersizes \ --folder 'INBOX.yop.yap' \ --sep1 '/' \ --regextrans2 's,/,_,' @@ -1133,7 +1176,7 @@ ll_regextrans2_subfolder() --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ --justfolders \ - --nofoldersize \ + --nofoldersizes \ --folder 'INBOX.yop.yap' \ --prefix1 'INBOX.yop.' \ --regextrans2 's,^${h2_prefix}(.*),${h2_prefix}FOO${h2_sep}$1,' --dry @@ -1149,7 +1192,7 @@ ll_regextrans2_remove_space() --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ --justfolders \ - --nofoldersize \ + --nofoldersizes \ --folder 'INBOX.yop.y p' \ --regextrans2 's, ,,' \ --dry @@ -1171,7 +1214,7 @@ ll_regextrans2_archive_per_month() --passfile1 ../../var/pass/secret.tata \ --host2 $HOST2 --user2 titi \ --passfile2 ../../var/pass/secret.titi \ - --nofoldersize \ + --nofoldersizes \ --search "SENTSINCE 1-$month-$year SENTBEFORE 30-$month-$year" \ --regextrans2 "s{.*}{INBOX.Archive.$year.$month_n}" } @@ -1647,6 +1690,17 @@ ll_delete2_uidexpunge2_implicit() { } +ll_delete2duplicates() { + can_send && sendtestmessage titi + can_send && sendtestmessage tata + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX \ + --delete2duplicates --uidexpunge2 +} ll_delete2_dev() { @@ -1898,6 +1952,22 @@ xxxxx_gmail_7() { } +xxxxx_gmail_8_xlist() { + + ! ping -c1 imap.gmail.com || $CMD_PERL ./imapsync \ + --host1 $HOST2 \ + --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 imap.gmail.com \ + --ssl2 \ + --user2 gilles.lamiral@gmail.com \ + --passfile2 ../../var/pass/secret.gilles_gmail \ + --foldersizes \ + --folder INBOX +} + + + gmail_xxxxx() { ! ping -c1 imap.gmail.com || $CMD_PERL ./imapsync \ @@ -2180,24 +2250,26 @@ ll_nousecache() { --folder INBOX } -ll_useuid_usecache() -{ + +ll_usecache_INBOX() { if can_send; then sendtestmessage else : fi - $CMD_PERL ./imapsync \ - --host1 $HOST1 --user1 tata \ - --passfile1 ../../var/pass/secret.tata \ - --host2 $HOST2 --user2 titi \ - --passfile2 ../../var/pass/secret.titi \ - --folder INBOX \ - --delete2 --expunge2 \ - --useuid - echo 'rm /home/vmail/titi/.yop.yap/cur/*' + + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --usecache --foldersizes \ + --folder INBOX } + + + ll_usecache_noheader() { if can_send; then sendtestmessage @@ -2246,6 +2318,23 @@ ll_usecache_debugcache_useuid() { --folder INBOX --useheader '' --debugcache --useuid } +ll_useuid_usecache() +{ + if can_send; then + sendtestmessage + else + : + fi + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX \ + --delete2 --expunge2 \ + --useuid + echo 'rm /home/vmail/titi/.yop.yap/cur/*' +} ll_useuid() { @@ -2885,10 +2974,12 @@ option_tests option_tests_debug option_bad_delete2 passwords_masked +passwords_not_masked first_sync_dry first_sync locallocal pidfile +ll_pidfilelocking justbanner nomodules_version xxxxx_gmail